{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import multiprocessing as mp\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from collections import namedtuple\n",
    "\n",
    "from PIL import Image\n",
    "import os\n",
    "import os.path\n",
    "import errno\n",
    "import codecs\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.cuda.device_count() 4\n",
      "torch.cuda.current_device() 0\n",
      "torch.cuda.current_device() 2\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "np.random.seed(0)\n",
    "print(\"torch.cuda.device_count()\", torch.cuda.device_count())\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())\n",
    "torch.cuda.set_device(2)\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_ranks(x):\n",
    "  \"\"\"\n",
    "  Returns ranks in [0, len(x))\n",
    "  Note: This is different from scipy.stats.rankdata, which returns ranks in [1, len(x)].\n",
    "  (https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py)\n",
    "  \"\"\"\n",
    "  assert x.ndim == 1\n",
    "  ranks = np.empty(len(x), dtype=int)\n",
    "  ranks[x.argsort()] = np.arange(len(x))\n",
    "  return ranks\n",
    "\n",
    "def compute_centered_ranks(x):\n",
    "  \"\"\"\n",
    "  https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py\n",
    "  \"\"\"\n",
    "  y = compute_ranks(x.ravel()).reshape(x.shape).astype(np.float32)\n",
    "  y /= (x.size - 1)\n",
    "  y -= .5\n",
    "  return y\n",
    "\n",
    "def compute_weight_decay(weight_decay, model_param_list):\n",
    "  model_param_grid = np.array(model_param_list)\n",
    "  return - weight_decay * np.mean(model_param_grid * model_param_grid, axis=1)\n",
    "\n",
    "class CMAES:\n",
    "  '''CMA-ES wrapper.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               popsize=255):          # population size\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.popsize = popsize\n",
    "\n",
    "    self.solutions = None\n",
    "\n",
    "    import cma\n",
    "    self.es = cma.CMAEvolutionStrategy( self.num_params * [0],\n",
    "                                        self.sigma_init,\n",
    "                                        {'popsize': self.popsize})\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.es.result()[6]\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    self.solutions = np.array(self.es.ask())\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    reward_table = reward_table_result\n",
    "    self.es.tell(self.solutions, (-reward_table).tolist()) # convert minimizer to maximizer.\n",
    "\n",
    "  def done(self):\n",
    "    return self.es.stop()\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.es.result()[5] # mean solution, presumably better with noise\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.es.result()[0] # best evaluated solution\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    r = self.es.result()\n",
    "    return (r[0], -r[1], -r[1], r[6])\n",
    "\n",
    "class SimpleES:\n",
    "  '''Simple Evolution Strategies.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               sigma_alpha=0.20,      # learning rate for standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               done_threshold=1e-6,   # threshold when we say we are done\n",
    "               average_baseline=True, # set baseline to average of batch\n",
    "               forget_best=True):     # only use the best from latest generation\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_alpha = sigma_alpha\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "    self.average_baseline = average_baseline\n",
    "    if self.average_baseline:\n",
    "      assert (self.popsize & 2), \"Population size must be even\"\n",
    "      self.batch_size = int(self.popsize / 2)\n",
    "    else:\n",
    "      assert (self.popsize & 1), \"Population size must be odd\"\n",
    "      self.batch_size = int((self.popsize - 1) / 2)\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "    self.forget_best = forget_best\n",
    "    self.batch_reward = np.zeros(self.batch_size * 2)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.sigma = np.ones(self.num_params) * self.sigma_init\n",
    "    self.curr_best_mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.batch_size, self.num_params) * self.sigma.reshape(1, self.num_params)\n",
    "    self.epsilon_full = np.concatenate([self.epsilon, - self.epsilon])\n",
    "    if self.average_baseline:\n",
    "      epsilon = self.epsilon_full\n",
    "    else:\n",
    "      # first population is mu, then positive epsilon, then negative epsilon\n",
    "      epsilon = np.concatenate([np.zeros((1, self.num_params)), self.epsilon_full])\n",
    "    solutions = self.mu.reshape(1, self.num_params) + epsilon\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    reward_table = reward_table_result\n",
    "\n",
    "    reward_offset = 1\n",
    "    if self.average_baseline:\n",
    "      b = np.mean(reward_table)\n",
    "      reward_offset = 0\n",
    "    else:\n",
    "      b = reward_table[0] # baseline\n",
    "      \n",
    "    reward = reward_table[reward_offset:]\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    if (best_reward > b or self.average_baseline):\n",
    "      best_mu = self.mu + self.epsilon_full[idx[0]]\n",
    "      best_reward = reward[idx[0]]\n",
    "    else:\n",
    "      best_mu = self.mu\n",
    "      best_reward = b\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # adaptive sigma\n",
    "    # normalization\n",
    "    stdev_reward = reward.std()\n",
    "    epsilon = self.epsilon\n",
    "    sigma = self.sigma\n",
    "    S = ((epsilon * epsilon - (sigma * sigma).reshape(1, self.num_params)) / sigma.reshape(1, self.num_params))\n",
    "    reward_avg = (reward[:self.batch_size] + reward[self.batch_size:]) / 2.0\n",
    "    rS = reward_avg - b\n",
    "    delta_sigma = (np.dot(rS, S)) / (2 * self.batch_size * stdev_reward)\n",
    "\n",
    "    # move mean to the average of the best idx means\n",
    "    self.mu += self.epsilon_full[idx].mean(axis=0)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    change_sigma = self.sigma_alpha * delta_sigma\n",
    "    change_sigma = np.minimum(change_sigma, self.sigma)\n",
    "    change_sigma = np.maximum(change_sigma, - 0.5 * self.sigma)\n",
    "    self.sigma += change_sigma\n",
    "    self.sigma[self.sigma > self.sigma_limit] *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class SimpleGA:\n",
    "  '''Simple Genetic Algorithm.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.1,        # initial standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               forget_best=False,     # forget the historical best elites\n",
    "               done_threshold=1e-6):  # threshold when we say we are done\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "\n",
    "    self.sigma = self.sigma_init\n",
    "    self.elite_params = np.zeros((self.elite_popsize, self.num_params))\n",
    "    self.elite_rewards = np.zeros(self.elite_popsize)\n",
    "    self.best_param = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_iteration = True\n",
    "    self.forget_best = forget_best\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    return self.sigma # same sigma for all parameters.\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.popsize, self.num_params) * self.sigma\n",
    "    solutions = []\n",
    "    \n",
    "    def mate(a, b):\n",
    "      c = np.copy(a)\n",
    "      idx = np.where(np.random.rand((c.size)) > 0.5)\n",
    "      c[idx] = b[idx]\n",
    "      return c\n",
    "    \n",
    "    elite_range = range(self.elite_popsize)\n",
    "    for i in range(self.popsize):\n",
    "      idx_a = np.random.choice(elite_range)\n",
    "      idx_b = np.random.choice(elite_range)\n",
    "      child_params = mate(self.elite_params[idx_a], self.elite_params[idx_b])\n",
    "      solutions.append(child_params + self.epsilon[i])\n",
    "\n",
    "    solutions = np.array(solutions)\n",
    "    self.solutions = solutions\n",
    "\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "    \n",
    "    if (not self.forget_best or self.first_iteration):\n",
    "      reward = reward_table_result\n",
    "      solution = self.solutions\n",
    "    else:\n",
    "      reward = np.concatenate([reward_table_result, self.elite_rewards])\n",
    "      solution = np.concatenate([self.solutions, self.elite_params])\n",
    "\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    self.elite_rewards = reward[idx]\n",
    "    self.elite_params = solution[idx]\n",
    "\n",
    "    self.curr_best_reward = self.elite_rewards[0]\n",
    "    \n",
    "    if self.first_iteration or (self.curr_best_reward > self.best_reward):\n",
    "      self.first_iteration = False\n",
    "      self.best_reward = self.elite_rewards[0]\n",
    "      self.best_param = np.copy(self.elite_params[0])\n",
    "\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.elite_params[0]\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_param\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_param, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class OpenES:\n",
    "  ''' Basic Version of OpenAI Evolution Strategies.'''\n",
    "  def __init__(self, num_params,             # number of model parameters\n",
    "               sigma_init=0.1,               # initial standard deviation\n",
    "               sigma_decay=0.999,            # anneal standard deviation\n",
    "               sigma_limit=0.01,             # stop annealing if less than this\n",
    "               learning_rate=0.001,          # learning rate for standard deviation\n",
    "               learning_rate_decay = 0.9999, # annealing the learning rate\n",
    "               learning_rate_limit = 0.001,  # stop annealing learning rate\n",
    "               popsize=255,                  # population size\n",
    "               antithetic=False,             # whether to use antithetic sampling\n",
    "               forget_best=True):           # forget historical best\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma = sigma_init\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.learning_rate = learning_rate\n",
    "    self.learning_rate_decay = learning_rate_decay\n",
    "    self.learning_rate_limit = learning_rate_limit\n",
    "    self.popsize = popsize\n",
    "    self.antithetic = antithetic\n",
    "    if self.antithetic:\n",
    "      assert (self.popsize & 2), \"Population size must be even\"\n",
    "      self.half_popsize = int(self.popsize / 2)\n",
    "\n",
    "    self.reward = np.zeros(self.popsize)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.forget_best = forget_best\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    if self.antithetic:\n",
    "      self.epsilon_half = np.random.randn(self.half_popsize, self.num_params)\n",
    "      self.epsilon = np.concatenate([self.epsilon_half, - self.epsilon_half])\n",
    "    else:\n",
    "      self.epsilon = np.random.randn(self.popsize, self.num_params)\n",
    "\n",
    "    self.solutions = self.mu.reshape(1, self.num_params) + self.epsilon * self.sigma\n",
    "\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    idx = np.argsort(reward)[::-1]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    best_mu = self.solutions[idx[0]]\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # main bit:\n",
    "    # standardize the rewards to have a gaussian distribution\n",
    "    normalized_reward = (reward - np.mean(reward)) / np.std(reward)\n",
    "    self.mu += self.learning_rate/(self.popsize*self.sigma)*np.dot(self.epsilon.T, normalized_reward)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "    if (self.learning_rate > self.learning_rate_limit):\n",
    "      self.learning_rate *= self.learning_rate_decay\n",
    "\n",
    "  def done(self):\n",
    "    return False\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Args = namedtuple('Args', ['batch_size', 'test_batch_size', 'epochs', 'lr', 'cuda', 'seed', 'log_interval'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = Args(batch_size=100, test_batch_size=1000, epochs=30, lr=0.001, cuda=True, seed=0, log_interval=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "  torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "valid_loader = train_loader\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=False, transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "  def __init__(self):\n",
    "    super(Net, self).__init__()\n",
    "    self.num_filter1 = 8\n",
    "    self.num_filter2 = 16\n",
    "    self.num_padding = 2\n",
    "    # input is 28x28\n",
    "    # padding=2 for same padding\n",
    "    self.conv1 = nn.Conv2d(1, self.num_filter1, 5, padding=self.num_padding)\n",
    "    # feature map size is 14*14 by pooling\n",
    "    # padding=2 for same padding\n",
    "    self.conv2 = nn.Conv2d(self.num_filter1, self.num_filter2, 5, padding=self.num_padding)\n",
    "    # feature map size is 7*7 by pooling\n",
    "    self.fc = nn.Linear(self.num_filter2*7*7, 10)\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "    x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "    x = x.view(-1, self.num_filter2*7*7)   # reshape Variable\n",
    "    x = self.fc(x)\n",
    "    return F.log_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NPOPULATION = 101\n",
    "weight_decay_coef = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "models = []\n",
    "for i in range(NPOPULATION):\n",
    "  model = Net()\n",
    "  if args.cuda:\n",
    "    model.cuda()\n",
    "  model.eval()\n",
    "  models.append(model)\n",
    "'''\n",
    "\n",
    "model = Net()\n",
    "if args.cuda:\n",
    "  model.cuda()\n",
    "\n",
    "orig_model = copy.deepcopy(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11274\n"
     ]
    }
   ],
   "source": [
    "# get init params\n",
    "orig_params = []\n",
    "model_shapes = []\n",
    "for param in orig_model.parameters():\n",
    "  p = param.data.cpu().numpy()\n",
    "  model_shapes.append(p.shape)\n",
    "  orig_params.append(p.flatten())\n",
    "orig_params_flat = np.concatenate(orig_params)\n",
    "NPARAMS = len(orig_params_flat)\n",
    "print(NPARAMS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_model(flat_param, model, model_shapes):\n",
    "  idx = 0\n",
    "  i = 0\n",
    "  for param in model.parameters():\n",
    "    delta = np.product(model_shapes[i])\n",
    "    block = flat_param[idx:idx+delta]\n",
    "    block = np.reshape(block, model_shapes[i])\n",
    "    i += 1\n",
    "    idx += delta\n",
    "    block_data = torch.from_numpy(block).float()\n",
    "    if args.cuda:\n",
    "      block_data = block_data.cuda()\n",
    "    param.data = block_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(model, test_loader, print_mode=True, return_loss=False):\n",
    "  model.eval()\n",
    "  test_loss = 0\n",
    "  correct = 0\n",
    "  for data, target in test_loader:\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data, volatile=True), Variable(target)\n",
    "    output = model(data)\n",
    "    test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss\n",
    "    pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "    correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "  test_loss /= len(test_loader.dataset)\n",
    "  acc = correct / len(test_loader.dataset)\n",
    "  \n",
    "  if print_mode:\n",
    "    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\\n'.format(\n",
    "      test_loss, correct, len(test_loader.dataset),\n",
    "      100. * acc))\n",
    "  \n",
    "  if return_loss:\n",
    "    return test_loss\n",
    "  return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "es = SimpleES(NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.999,\n",
    "              sigma_alpha=0.2,\n",
    "              sigma_limit=0.001,\n",
    "              elite_ratio=0.1,\n",
    "              average_baseline=False,\n",
    "              forget_best=True\n",
    "             )\n",
    "\"\"\"\n",
    "es = SimpleGA(NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              forget_best=False,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.9999,\n",
    "              sigma_limit=0.01\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def worker(procnum, model, solution, data, target, send_end):\n",
    "  update_model(solution, model, model_shapes)\n",
    "  output = model(data)\n",
    "  loss = F.nll_loss(output, target)\n",
    "  reward = - loss.data[0]\n",
    "  send_end.send(reward)\n",
    "\n",
    "def batch_simulation(model_list, solutions, data, target, process_count):\n",
    "  jobs = []\n",
    "  pipe_list = []\n",
    "\n",
    "  for i in range(process_count):\n",
    "    recv_end, send_end = mp.Pipe(False)\n",
    "    p = mp.Process(target=worker, args=(i, model_list[i], solutions[i], data, target, send_end))\n",
    "    jobs.append(p)\n",
    "    pipe_list.append(recv_end)\n",
    "\n",
    "  for p in jobs:\n",
    "    p.start()\n",
    "\n",
    "  for p in jobs:\n",
    "    p.join()\n",
    "\n",
    "  result_list = [x.recv() for x in pipe_list]\n",
    "  return np.array(result_list)\n",
    "\n",
    "\n",
    "def batch_simulation_sequential(model_list, solutions, data, target, process_count):\n",
    "  result_list = []\n",
    "  for i in range(process_count):\n",
    "    update_model(solutions[i], model_list[i], model_shapes)\n",
    "    output = model_list[i](data)\n",
    "    loss = F.nll_loss(output, target)\n",
    "    reward = - loss.data[0]\n",
    "    result_list.append(reward)\n",
    "  return np.array(result_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 -2.30060434341 -0.000112202587902 0.01\n",
      "1 50 -1.20309865475 -0.000112202587902 0.01\n",
      "1 100 -1.10738801956 -0.000112202587902 0.01\n",
      "1 150 -0.919825077057 -0.000112202587902 0.01\n",
      "1 200 -0.961853086948 -0.000112202587902 0.01\n",
      "1 250 -0.976065456867 -0.000112202587902 0.01\n",
      "1 300 -0.945845663548 -0.000112202587902 0.01\n",
      "1 350 -1.14225947857 -0.000112202587902 0.01\n",
      "1 400 -0.837524473667 -0.000112202587902 0.01\n",
      "1 450 -0.634921133518 -0.000112202587902 0.01\n",
      "1 500 -0.948218405247 -0.000112202587902 0.01\n",
      "1 550 -0.961603403091 -0.000112202587902 0.01\n",
      "valid_acc 80.205\n",
      "best valid_acc 80.205\n",
      "2 0 -0.757367372513 -0.000112202587902 0.01\n",
      "2 50 -0.732529580593 -0.000112202587902 0.01\n",
      "2 100 -1.03761339188 -0.000112202587902 0.01\n",
      "2 150 -1.2254191637 -0.000112202587902 0.01\n",
      "2 200 -0.758734345436 -0.000112202587902 0.01\n",
      "2 250 -0.64583837986 -0.000112202587902 0.01\n",
      "2 300 -1.1819319725 -0.000112202587902 0.01\n",
      "2 350 -0.632461309433 -0.000112202587902 0.01\n",
      "2 400 -0.862645924091 -0.000112202587902 0.01\n",
      "2 450 -0.408658742905 -0.000112202587902 0.01\n",
      "2 500 -0.791494131088 -0.000112202587902 0.01\n",
      "2 550 -1.36785781384 -0.000112202587902 0.01\n",
      "valid_acc 79.50333333333333\n",
      "3 0 -1.12649822235 -0.000112202587902 0.01\n",
      "3 50 -1.07467222214 -0.000112202587902 0.01\n",
      "3 100 -1.16048204899 -0.000112202587902 0.01\n",
      "3 150 -0.881195604801 -0.000112202587902 0.01\n",
      "3 200 -0.785757660866 -0.000112202587902 0.01\n",
      "3 250 -0.796030521393 -0.000112202587902 0.01\n",
      "3 300 -1.21759176254 -0.000112202587902 0.01\n",
      "3 350 -1.23741269112 -0.000112202587902 0.01\n",
      "3 400 -1.56565356255 -0.000112202587902 0.01\n",
      "3 450 -0.70002502203 -0.000112202587902 0.01\n",
      "3 500 -0.983105778694 -0.000112202587902 0.01\n",
      "3 550 -0.824114441872 -0.000112202587902 0.01\n",
      "valid_acc 71.07833333333333\n",
      "4 0 -1.19405865669 -0.000112202587902 0.01\n",
      "4 50 -1.03346967697 -0.000112202587902 0.01\n",
      "4 100 -0.830691695213 -0.000112202587902 0.01\n",
      "4 150 -0.722347199917 -0.000112202587902 0.01\n",
      "4 200 -1.13884305954 -0.000112202587902 0.01\n",
      "4 250 -1.32381403446 -0.000112202587902 0.01\n",
      "4 300 -1.14037740231 -0.000112202587902 0.01\n",
      "4 350 -1.46665835381 -0.000112202587902 0.01\n",
      "4 400 -1.14534795284 -0.000112202587902 0.01\n",
      "4 450 -0.812051713467 -0.000112202587902 0.01\n",
      "4 500 -0.707704722881 -0.000112202587902 0.01\n",
      "4 550 -0.944455385208 -0.000112202587902 0.01\n",
      "valid_acc 75.51333333333334\n",
      "5 0 -0.543698847294 -0.000112202587902 0.01\n",
      "5 50 -0.794459819794 -0.000112202587902 0.01\n",
      "5 100 -1.00134730339 -0.000112202587902 0.01\n",
      "5 150 -0.573217868805 -0.000112202587902 0.01\n",
      "5 200 -0.635794401169 -0.000112202587902 0.01\n",
      "5 250 -0.69489210844 -0.000112202587902 0.01\n",
      "5 300 -1.03711605072 -0.000112202587902 0.01\n",
      "5 350 -1.09784781933 -0.000112202587902 0.01\n",
      "5 400 -1.04052746296 -0.000112202587902 0.01\n",
      "5 450 -0.941569745541 -0.000112202587902 0.01\n",
      "5 500 -0.917918086052 -0.000112202587902 0.01\n",
      "5 550 -0.653759717941 -0.000112202587902 0.01\n",
      "valid_acc 72.695\n",
      "6 0 -1.04216015339 -0.000112202587902 0.01\n",
      "6 50 -1.16818380356 -0.000112202587902 0.01\n",
      "6 100 -1.07018506527 -0.000112202587902 0.01\n",
      "6 150 -0.921358346939 -0.000112202587902 0.01\n",
      "6 200 -0.917170464993 -0.000112202587902 0.01\n",
      "6 250 -1.40848362446 -0.000112202587902 0.01\n",
      "6 300 -0.967659890652 -0.000112202587902 0.01\n",
      "6 350 -0.760281682014 -0.000112202587902 0.01\n",
      "6 400 -0.489991039038 -0.000112202587902 0.01\n",
      "6 450 -0.828100383282 -0.000112202587902 0.01\n",
      "6 500 -0.879557192326 -0.000112202587902 0.01\n",
      "6 550 -1.31469345093 -0.000112202587902 0.01\n",
      "valid_acc 71.85000000000001\n",
      "7 0 -1.44827997684 -0.000112202587902 0.01\n",
      "7 50 -1.28055632114 -0.000112202587902 0.01\n",
      "7 100 -1.1802520752 -0.000112202587902 0.01\n",
      "7 150 -1.12539446354 -0.000112202587902 0.01\n",
      "7 200 -0.676623702049 -0.000112202587902 0.01\n",
      "7 250 -1.13439238071 -0.000112202587902 0.01\n",
      "7 300 -1.02525401115 -0.000112202587902 0.01\n",
      "7 350 -0.828598618507 -0.000112202587902 0.01\n",
      "7 400 -0.805998682976 -0.000112202587902 0.01\n",
      "7 450 -1.2369941473 -0.000112202587902 0.01\n",
      "7 500 -1.50523912907 -0.000112202587902 0.01\n",
      "7 550 -0.676962971687 -0.000112202587902 0.01\n",
      "valid_acc 72.255\n",
      "8 0 -0.985406279564 -0.000112202587902 0.01\n",
      "8 50 -0.696985781193 -0.000112202587902 0.01\n",
      "8 100 -0.731779634953 -0.000112202587902 0.01\n",
      "8 150 -1.03845322132 -0.000112202587902 0.01\n",
      "8 200 -0.734666287899 -0.000112202587902 0.01\n",
      "8 250 -0.670766234398 -0.000112202587902 0.01\n",
      "8 300 -1.45708048344 -0.000112202587902 0.01\n",
      "8 350 -1.13521873951 -0.000112202587902 0.01\n",
      "8 400 -1.0879380703 -0.000112202587902 0.01\n",
      "8 450 -0.923160493374 -0.000112202587902 0.01\n",
      "8 500 -0.983298540115 -0.000112202587902 0.01\n",
      "8 550 -1.53950345516 -0.000112202587902 0.01\n",
      "valid_acc 74.62333333333333\n",
      "9 0 -0.660493314266 -0.000112202587902 0.01\n",
      "9 50 -1.06333255768 -0.000112202587902 0.01\n",
      "9 100 -0.960911512375 -0.000112202587902 0.01\n",
      "9 150 -0.695252895355 -0.000112202587902 0.01\n",
      "9 200 -1.3040804863 -0.000112202587902 0.01\n",
      "9 250 -0.92146474123 -0.000112202587902 0.01\n",
      "9 300 -1.19945371151 -0.000112202587902 0.01\n",
      "9 350 -0.663622617722 -0.000112202587902 0.01\n",
      "9 400 -0.779250383377 -0.000112202587902 0.01\n",
      "9 450 -0.901264190674 -0.000112202587902 0.01\n",
      "9 500 -0.664028167725 -0.000112202587902 0.01\n",
      "9 550 -0.769926965237 -0.000112202587902 0.01\n",
      "valid_acc 73.71333333333332\n",
      "10 0 -1.05525112152 -0.000112202587902 0.01\n",
      "10 50 -0.793425321579 -0.000112202587902 0.01\n",
      "10 100 -0.921313405037 -0.000112202587902 0.01\n",
      "10 150 -1.10802280903 -0.000112202587902 0.01\n",
      "10 200 -1.20507764816 -0.000112202587902 0.01\n",
      "10 250 -1.18751895428 -0.000112202587902 0.01\n",
      "10 300 -1.35988712311 -0.000112202587902 0.01\n",
      "10 350 -0.969793021679 -0.000112202587902 0.01\n",
      "10 400 -0.993719279766 -0.000112202587902 0.01\n",
      "10 450 -0.757110118866 -0.000112202587902 0.01\n",
      "10 500 -0.743792951107 -0.000112202587902 0.01\n",
      "10 550 -1.0354398489 -0.000112202587902 0.01\n",
      "valid_acc 76.74333333333333\n",
      "11 0 -0.981331050396 -0.000112202587902 0.01\n",
      "11 50 -0.565507531166 -0.000112202587902 0.01\n",
      "11 100 -0.581033885479 -0.000112202587902 0.01\n",
      "11 150 -0.927199006081 -0.000112202587902 0.01\n",
      "11 200 -0.855668723583 -0.000112202587902 0.01\n",
      "11 250 -0.914460301399 -0.000112202587902 0.01\n",
      "11 300 -0.969400048256 -0.000112202587902 0.01\n",
      "11 350 -1.14424657822 -0.000112202587902 0.01\n",
      "11 400 -1.33686900139 -0.000112202587902 0.01\n",
      "11 450 -1.02357888222 -0.000112202587902 0.01\n",
      "11 500 -0.45202049613 -0.000112202587902 0.01\n",
      "11 550 -0.877157568932 -0.000112202587902 0.01\n",
      "valid_acc 77.215\n",
      "12 0 -0.880722820759 -0.000112202587902 0.01\n",
      "12 50 -0.88317078352 -0.000112202587902 0.01\n",
      "12 100 -1.05834078789 -0.000112202587902 0.01\n",
      "12 150 -0.951174497604 -0.000112202587902 0.01\n",
      "12 200 -1.53143787384 -0.000112202587902 0.01\n",
      "12 250 -0.952684640884 -0.000112202587902 0.01\n",
      "12 300 -0.437341153622 -0.000112202587902 0.01\n",
      "12 350 -0.727446496487 -0.000112202587902 0.01\n",
      "12 400 -0.416227579117 -0.000112202587902 0.01\n",
      "12 450 -0.86175942421 -0.000112202587902 0.01\n",
      "12 500 -0.736036658287 -0.000112202587902 0.01\n",
      "12 550 -0.675183773041 -0.000112202587902 0.01\n",
      "valid_acc 74.71166666666666\n",
      "13 0 -0.977424681187 -0.000112202587902 0.01\n",
      "13 50 -1.00105214119 -0.000112202587902 0.01\n",
      "13 100 -0.965642631054 -0.000112202587902 0.01\n",
      "13 150 -0.743781328201 -0.000112202587902 0.01\n",
      "13 200 -0.780153334141 -0.000112202587902 0.01\n",
      "13 250 -0.943228900433 -0.000112202587902 0.01\n",
      "13 300 -1.33343362808 -0.000112202587902 0.01\n",
      "13 350 -0.995757162571 -0.000112202587902 0.01\n",
      "13 400 -1.00366592407 -0.000112202587902 0.01\n",
      "13 450 -0.821783363819 -0.000112202587902 0.01\n",
      "13 500 -1.0256472826 -0.000112202587902 0.01\n",
      "13 550 -0.830449700356 -0.000112202587902 0.01\n",
      "valid_acc 77.35833333333333\n",
      "14 0 -0.925390601158 -0.000112202587902 0.01\n",
      "14 50 -0.85819542408 -0.000112202587902 0.01\n",
      "14 100 -1.35226261616 -0.000112202587902 0.01\n",
      "14 150 -1.40766358376 -0.000112202587902 0.01\n",
      "14 200 -0.870977938175 -0.000112202587902 0.01\n",
      "14 250 -1.0860543251 -0.000112202587902 0.01\n",
      "14 300 -0.721102952957 -0.000112202587902 0.01\n",
      "14 350 -1.04604542255 -0.000112202587902 0.01\n",
      "14 400 -1.30891907215 -0.000112202587902 0.01\n",
      "14 450 -0.797980487347 -0.000112202587902 0.01\n",
      "14 500 -0.826778054237 -0.000112202587902 0.01\n",
      "14 550 -0.944269120693 -0.000112202587902 0.01\n",
      "valid_acc 75.75166666666667\n",
      "15 0 -0.783657073975 -0.000112202587902 0.01\n",
      "15 50 -0.503866076469 -0.000112202587902 0.01\n",
      "15 100 -0.761872768402 -0.000112202587902 0.01\n",
      "15 150 -1.37296259403 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 200 -0.645030379295 -0.000112202587902 0.01\n",
      "15 250 -0.789413750172 -0.000112202587902 0.01\n",
      "15 300 -0.825545430183 -0.000112202587902 0.01\n",
      "15 350 -1.29622197151 -0.000112202587902 0.01\n",
      "15 400 -0.878007531166 -0.000112202587902 0.01\n",
      "15 450 -0.836985468864 -0.000112202587902 0.01\n",
      "15 500 -0.520756959915 -0.000112202587902 0.01\n",
      "15 550 -0.939542472363 -0.000112202587902 0.01\n",
      "valid_acc 73.2\n",
      "16 0 -1.11171543598 -0.000112202587902 0.01\n",
      "16 50 -1.19556856155 -0.000112202587902 0.01\n",
      "16 100 -0.803073704243 -0.000112202587902 0.01\n",
      "16 150 -0.825310111046 -0.000112202587902 0.01\n",
      "16 200 -0.85929608345 -0.000112202587902 0.01\n",
      "16 250 -0.854680776596 -0.000112202587902 0.01\n",
      "16 300 -1.10111641884 -0.000112202587902 0.01\n",
      "16 350 -1.03547465801 -0.000112202587902 0.01\n",
      "16 400 -1.06607413292 -0.000112202587902 0.01\n",
      "16 450 -1.3802074194 -0.000112202587902 0.01\n",
      "16 500 -1.16331291199 -0.000112202587902 0.01\n",
      "16 550 -0.873036324978 -0.000112202587902 0.01\n",
      "valid_acc 74.225\n",
      "17 0 -0.676514804363 -0.000112202587902 0.01\n",
      "17 50 -1.00033533573 -0.000112202587902 0.01\n",
      "17 100 -1.00532245636 -0.000112202587902 0.01\n",
      "17 150 -1.0549826622 -0.000112202587902 0.01\n",
      "17 200 -1.23300504684 -0.000112202587902 0.01\n",
      "17 250 -0.800634086132 -0.000112202587902 0.01\n",
      "17 300 -1.25241088867 -0.000112202587902 0.01\n",
      "17 350 -0.762112498283 -0.000112202587902 0.01\n",
      "17 400 -0.483923375607 -0.000112202587902 0.01\n",
      "17 450 -0.909099876881 -0.000112202587902 0.01\n",
      "17 500 -1.0079061985 -0.000112202587902 0.01\n",
      "17 550 -0.775996685028 -0.000112202587902 0.01\n",
      "valid_acc 76.03999999999999\n",
      "18 0 -0.513692975044 -0.000112202587902 0.01\n",
      "18 50 -0.542120695114 -0.000112202587902 0.01\n",
      "18 100 -0.860245049 -0.000112202587902 0.01\n",
      "18 150 -0.935307145119 -0.000112202587902 0.01\n",
      "18 200 -0.841389775276 -0.000112202587902 0.01\n",
      "18 250 -0.945169508457 -0.000112202587902 0.01\n",
      "18 300 -0.619059205055 -0.000112202587902 0.01\n",
      "18 350 -1.14068293571 -0.000112202587902 0.01\n",
      "18 400 -0.662838816643 -0.000112202587902 0.01\n",
      "18 450 -0.666875839233 -0.000112202587902 0.01\n",
      "18 500 -0.929730653763 -0.000112202587902 0.01\n",
      "18 550 -0.788258492947 -0.000112202587902 0.01\n",
      "valid_acc 75.35833333333333\n",
      "19 0 -0.820223331451 -0.000112202587902 0.01\n",
      "19 50 -1.1356344223 -0.000112202587902 0.01\n",
      "19 100 -1.04373180866 -0.000112202587902 0.01\n",
      "19 150 -0.840634822845 -0.000112202587902 0.01\n",
      "19 200 -0.792300343513 -0.000112202587902 0.01\n",
      "19 250 -1.14878451824 -0.000112202587902 0.01\n",
      "19 300 -0.764628350735 -0.000112202587902 0.01\n",
      "19 350 -0.902029633522 -0.000112202587902 0.01\n",
      "19 400 -0.725403726101 -0.000112202587902 0.01\n",
      "19 450 -0.643955290318 -0.000112202587902 0.01\n",
      "19 500 -0.734695374966 -0.000112202587902 0.01\n",
      "19 550 -0.417384952307 -0.000112202587902 0.01\n",
      "valid_acc 77.64333333333333\n",
      "20 0 -0.968840658665 -0.000112202587902 0.01\n",
      "20 50 -1.18582975864 -0.000112202587902 0.01\n",
      "20 100 -0.485206216574 -0.000112202587902 0.01\n",
      "20 150 -0.578970730305 -0.000112202587902 0.01\n",
      "20 200 -1.20345020294 -0.000112202587902 0.01\n",
      "20 250 -0.773482978344 -0.000112202587902 0.01\n",
      "20 300 -1.03908610344 -0.000112202587902 0.01\n",
      "20 350 -0.723359286785 -0.000112202587902 0.01\n",
      "20 400 -0.97548109293 -0.000112202587902 0.01\n",
      "20 450 -0.849772274494 -0.000112202587902 0.01\n",
      "20 500 -0.668652892113 -0.000112202587902 0.01\n",
      "20 550 -1.10201513767 -0.000112202587902 0.01\n",
      "valid_acc 76.16333333333334\n",
      "21 0 -1.18304276466 -0.000112202587902 0.01\n",
      "21 50 -0.823680579662 -0.000112202587902 0.01\n",
      "21 100 -0.74629431963 -0.000112202587902 0.01\n",
      "21 150 -1.1334297657 -0.000112202587902 0.01\n",
      "21 200 -1.1971116066 -0.000112202587902 0.01\n",
      "21 250 -0.813824832439 -0.000112202587902 0.01\n",
      "21 300 -1.21261715889 -0.000112202587902 0.01\n",
      "21 350 -0.787259817123 -0.000112202587902 0.01\n",
      "21 400 -0.836054325104 -0.000112202587902 0.01\n",
      "21 450 -0.741477668285 -0.000112202587902 0.01\n",
      "21 500 -1.21847093105 -0.000112202587902 0.01\n",
      "21 550 -1.02433109283 -0.000112202587902 0.01\n",
      "valid_acc 72.12166666666666\n",
      "22 0 -0.885460436344 -0.000112202587902 0.01\n",
      "22 50 -0.922540605068 -0.000112202587902 0.01\n",
      "22 100 -0.6695317626 -0.000112202587902 0.01\n",
      "22 150 -0.668830037117 -0.000112202587902 0.01\n",
      "22 200 -1.13704478741 -0.000112202587902 0.01\n",
      "22 250 -1.43873882294 -0.000112202587902 0.01\n",
      "22 300 -1.3207681179 -0.000112202587902 0.01\n",
      "22 350 -0.687303543091 -0.000112202587902 0.01\n",
      "22 400 -1.53958511353 -0.000112202587902 0.01\n",
      "22 450 -1.2965747118 -0.000112202587902 0.01\n",
      "22 500 -0.818981707096 -0.000112202587902 0.01\n",
      "22 550 -0.670953691006 -0.000112202587902 0.01\n",
      "valid_acc 73.88\n",
      "23 0 -0.882807612419 -0.000112202587902 0.01\n",
      "23 50 -0.973510146141 -0.000112202587902 0.01\n",
      "23 100 -0.742927193642 -0.000112202587902 0.01\n",
      "23 150 -1.06141674519 -0.000112202587902 0.01\n",
      "23 200 -0.642451941967 -0.000112202587902 0.01\n",
      "23 250 -0.965671896935 -0.000112202587902 0.01\n",
      "23 300 -0.684391617775 -0.000112202587902 0.01\n",
      "23 350 -0.910021722317 -0.000112202587902 0.01\n",
      "23 400 -1.06566238403 -0.000112202587902 0.01\n",
      "23 450 -0.660136282444 -0.000112202587902 0.01\n",
      "23 500 -1.03123891354 -0.000112202587902 0.01\n",
      "23 550 -0.773040473461 -0.000112202587902 0.01\n",
      "valid_acc 72.71833333333333\n",
      "24 0 -0.843104183674 -0.000112202587902 0.01\n",
      "24 50 -1.1594427824 -0.000112202587902 0.01\n",
      "24 100 -1.05991649628 -0.000112202587902 0.01\n",
      "24 150 -1.1576884985 -0.000112202587902 0.01\n",
      "24 200 -1.03034460545 -0.000112202587902 0.01\n",
      "24 250 -0.999156415462 -0.000112202587902 0.01\n",
      "24 300 -0.810526430607 -0.000112202587902 0.01\n",
      "24 350 -1.05572843552 -0.000112202587902 0.01\n",
      "24 400 -1.00987577438 -0.000112202587902 0.01\n",
      "24 450 -0.640587389469 -0.000112202587902 0.01\n",
      "24 500 -0.749646127224 -0.000112202587902 0.01\n",
      "24 550 -0.990852296352 -0.000112202587902 0.01\n",
      "valid_acc 72.45166666666667\n",
      "25 0 -0.734032273293 -0.000112202587902 0.01\n",
      "25 50 -0.857891321182 -0.000112202587902 0.01\n",
      "25 100 -0.607295870781 -0.000112202587902 0.01\n",
      "25 150 -0.83621019125 -0.000112202587902 0.01\n",
      "25 200 -0.688419818878 -0.000112202587902 0.01\n",
      "25 250 -1.07364702225 -0.000112202587902 0.01\n",
      "25 300 -1.05287539959 -0.000112202587902 0.01\n",
      "25 350 -0.908157348633 -0.000112202587902 0.01\n",
      "25 400 -1.17567873001 -0.000112202587902 0.01\n",
      "25 450 -0.894006192684 -0.000112202587902 0.01\n",
      "25 500 -1.04776871204 -0.000112202587902 0.01\n",
      "25 550 -1.13795161247 -0.000112202587902 0.01\n",
      "valid_acc 71.17666666666666\n",
      "26 0 -0.970011591911 -0.000112202587902 0.01\n",
      "26 50 -1.15542209148 -0.000112202587902 0.01\n",
      "26 100 -1.18899226189 -0.000112202587902 0.01\n",
      "26 150 -1.23159658909 -0.000112202587902 0.01\n",
      "26 200 -0.884483337402 -0.000112202587902 0.01\n",
      "26 250 -0.611949980259 -0.000112202587902 0.01\n",
      "26 300 -1.16272556782 -0.000112202587902 0.01\n",
      "26 350 -0.690988004208 -0.000112202587902 0.01\n",
      "26 400 -1.01502656937 -0.000112202587902 0.01\n",
      "26 450 -1.04596292973 -0.000112202587902 0.01\n",
      "26 500 -0.733855724335 -0.000112202587902 0.01\n",
      "26 550 -1.05696237087 -0.000112202587902 0.01\n",
      "valid_acc 72.20333333333333\n",
      "27 0 -0.907796502113 -0.000112202587902 0.01\n",
      "27 50 -0.925630331039 -0.000112202587902 0.01\n",
      "27 100 -1.09545600414 -0.000112202587902 0.01\n",
      "27 150 -0.792905449867 -0.000112202587902 0.01\n",
      "27 200 -0.859936833382 -0.000112202587902 0.01\n",
      "27 250 -0.713234841824 -0.000112202587902 0.01\n",
      "27 300 -1.07721245289 -0.000112202587902 0.01\n",
      "27 350 -0.929684400558 -0.000112202587902 0.01\n",
      "27 400 -1.10512173176 -0.000112202587902 0.01\n",
      "27 450 -1.08154535294 -0.000112202587902 0.01\n",
      "27 500 -0.976966023445 -0.000112202587902 0.01\n",
      "27 550 -0.730647265911 -0.000112202587902 0.01\n",
      "valid_acc 72.15\n",
      "28 0 -0.748273670673 -0.000112202587902 0.01\n",
      "28 50 -0.964345157146 -0.000112202587902 0.01\n",
      "28 100 -1.08736467361 -0.000112202587902 0.01\n",
      "28 150 -0.711858689785 -0.000112202587902 0.01\n",
      "28 200 -0.88948148489 -0.000112202587902 0.01\n",
      "28 250 -0.985235691071 -0.000112202587902 0.01\n",
      "28 300 -1.07155454159 -0.000112202587902 0.01\n",
      "28 350 -0.779705524445 -0.000112202587902 0.01\n",
      "28 400 -1.09900867939 -0.000112202587902 0.01\n",
      "28 450 -0.911158442497 -0.000112202587902 0.01\n",
      "28 500 -0.978824317455 -0.000112202587902 0.01\n",
      "28 550 -0.967768251896 -0.000112202587902 0.01\n",
      "valid_acc 69.075\n",
      "29 0 -0.972045600414 -0.000112202587902 0.01\n",
      "29 50 -0.967028200626 -0.000112202587902 0.01\n",
      "29 100 -1.26283538342 -0.000112202587902 0.01\n",
      "29 150 -0.904521644115 -0.000112202587902 0.01\n",
      "29 200 -0.731045544147 -0.000112202587902 0.01\n",
      "29 250 -0.850513160229 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29 300 -0.711340129375 -0.000112202587902 0.01\n",
      "29 350 -0.941682636738 -0.000112202587902 0.01\n",
      "29 400 -0.992071330547 -0.000112202587902 0.01\n",
      "29 450 -1.1413217783 -0.000112202587902 0.01\n",
      "29 500 -0.917519807816 -0.000112202587902 0.01\n",
      "29 550 -1.01626849174 -0.000112202587902 0.01\n",
      "valid_acc 66.39500000000001\n",
      "30 0 -0.616011798382 -0.000112202587902 0.01\n",
      "30 50 -0.956930339336 -0.000112202587902 0.01\n",
      "30 100 -0.976789116859 -0.000112202587902 0.01\n",
      "30 150 -0.94179046154 -0.000112202587902 0.01\n",
      "30 200 -0.921755433083 -0.000112202587902 0.01\n",
      "30 250 -0.907006382942 -0.000112202587902 0.01\n",
      "30 300 -0.835935354233 -0.000112202587902 0.01\n",
      "30 350 -0.976340353489 -0.000112202587902 0.01\n",
      "30 400 -1.08026218414 -0.000112202587902 0.01\n",
      "30 450 -1.57223188877 -0.000112202587902 0.01\n",
      "30 500 -1.0037535429 -0.000112202587902 0.01\n",
      "30 550 -0.827641427517 -0.000112202587902 0.01\n",
      "valid_acc 71.87833333333333\n",
      "31 0 -0.928116679192 -0.000112202587902 0.01\n",
      "31 50 -0.773678004742 -0.000112202587902 0.01\n",
      "31 100 -0.884632587433 -0.000112202587902 0.01\n",
      "31 150 -0.920069336891 -0.000112202587902 0.01\n",
      "31 200 -0.796917021275 -0.000112202587902 0.01\n",
      "31 250 -1.15995001793 -0.000112202587902 0.01\n",
      "31 300 -0.738288938999 -0.000112202587902 0.01\n",
      "31 350 -0.92398506403 -0.000112202587902 0.01\n",
      "31 400 -0.707104206085 -0.000112202587902 0.01\n",
      "31 450 -0.915246188641 -0.000112202587902 0.01\n",
      "31 500 -1.27417874336 -0.000112202587902 0.01\n",
      "31 550 -1.18676900864 -0.000112202587902 0.01\n",
      "valid_acc 73.84166666666667\n",
      "32 0 -0.94146668911 -0.000112202587902 0.01\n",
      "32 50 -0.646920859814 -0.000112202587902 0.01\n",
      "32 100 -1.00780451298 -0.000112202587902 0.01\n",
      "32 150 -1.04098236561 -0.000112202587902 0.01\n",
      "32 200 -0.754389286041 -0.000112202587902 0.01\n",
      "32 250 -1.04463541508 -0.000112202587902 0.01\n",
      "32 300 -0.839328587055 -0.000112202587902 0.01\n",
      "32 350 -1.12001597881 -0.000112202587902 0.01\n",
      "32 400 -0.946425616741 -0.000112202587902 0.01\n",
      "32 450 -0.944567084312 -0.000112202587902 0.01\n",
      "32 500 -0.82171869278 -0.000112202587902 0.01\n",
      "32 550 -0.911239922047 -0.000112202587902 0.01\n",
      "valid_acc 70.94333333333334\n",
      "33 0 -0.921973109245 -0.000112202587902 0.01\n",
      "33 50 -0.770638287067 -0.000112202587902 0.01\n",
      "33 100 -1.06953060627 -0.000112202587902 0.01\n",
      "33 150 -1.30836641788 -0.000112202587902 0.01\n",
      "33 200 -0.935111999512 -0.000112202587902 0.01\n",
      "33 250 -0.752932071686 -0.000112202587902 0.01\n",
      "33 300 -0.667919516563 -0.000112202587902 0.01\n",
      "33 350 -1.11641252041 -0.000112202587902 0.01\n",
      "33 400 -0.970733106136 -0.000112202587902 0.01\n",
      "33 450 -1.11487615108 -0.000112202587902 0.01\n",
      "33 500 -0.87027066946 -0.000112202587902 0.01\n",
      "33 550 -0.704297423363 -0.000112202587902 0.01\n",
      "valid_acc 71.485\n",
      "34 0 -1.05441975594 -0.000112202587902 0.01\n",
      "34 50 -0.98849105835 -0.000112202587902 0.01\n",
      "34 100 -0.928878486156 -0.000112202587902 0.01\n",
      "34 150 -0.910124838352 -0.000112202587902 0.01\n",
      "34 200 -0.834380567074 -0.000112202587902 0.01\n",
      "34 250 -0.748514175415 -0.000112202587902 0.01\n",
      "34 300 -0.765555560589 -0.000112202587902 0.01\n",
      "34 350 -0.962220549583 -0.000112202587902 0.01\n",
      "34 400 -1.05695867538 -0.000112202587902 0.01\n",
      "34 450 -0.775930047035 -0.000112202587902 0.01\n",
      "34 500 -0.915848374367 -0.000112202587902 0.01\n",
      "34 550 -0.649854123592 -0.000112202587902 0.01\n",
      "valid_acc 72.875\n",
      "35 0 -0.922276735306 -0.000112202587902 0.01\n",
      "35 50 -0.758308112621 -0.000112202587902 0.01\n",
      "35 100 -1.06449282169 -0.000112202587902 0.01\n",
      "35 150 -0.507400512695 -0.000112202587902 0.01\n",
      "35 200 -0.639629602432 -0.000112202587902 0.01\n",
      "35 250 -1.28092026711 -0.000112202587902 0.01\n",
      "35 300 -0.902974367142 -0.000112202587902 0.01\n",
      "35 350 -1.12265169621 -0.000112202587902 0.01\n",
      "35 400 -0.830138862133 -0.000112202587902 0.01\n",
      "35 450 -0.937622785568 -0.000112202587902 0.01\n",
      "35 500 -1.10010647774 -0.000112202587902 0.01\n",
      "35 550 -0.958362042904 -0.000112202587902 0.01\n",
      "valid_acc 76.505\n",
      "36 0 -0.899400889874 -0.000112202587902 0.01\n",
      "36 50 -0.840857565403 -0.000112202587902 0.01\n",
      "36 100 -0.667939603329 -0.000112202587902 0.01\n",
      "36 150 -0.653712153435 -0.000112202587902 0.01\n",
      "36 200 -0.711958944798 -0.000112202587902 0.01\n",
      "36 250 -0.835015237331 -0.000112202587902 0.01\n",
      "36 300 -0.778222501278 -0.000112202587902 0.01\n",
      "36 350 -1.21155464649 -0.000112202587902 0.01\n",
      "36 400 -0.960201442242 -0.000112202587902 0.01\n",
      "36 450 -0.811013460159 -0.000112202587902 0.01\n",
      "36 500 -0.840919017792 -0.000112202587902 0.01\n",
      "36 550 -1.06763541698 -0.000112202587902 0.01\n",
      "valid_acc 75.505\n",
      "37 0 -0.721715688705 -0.000112202587902 0.01\n",
      "37 50 -0.56914627552 -0.000112202587902 0.01\n",
      "37 100 -0.5103738904 -0.000112202587902 0.01\n",
      "37 150 -0.698937058449 -0.000112202587902 0.01\n",
      "37 200 -0.752378106117 -0.000112202587902 0.01\n",
      "37 250 -0.567685365677 -0.000112202587902 0.01\n",
      "37 300 -0.898069083691 -0.000112202587902 0.01\n",
      "37 350 -0.972969591618 -0.000112202587902 0.01\n",
      "37 400 -0.952286601067 -0.000112202587902 0.01\n",
      "37 450 -0.445189654827 -0.000112202587902 0.01\n",
      "37 500 -0.717525243759 -0.000112202587902 0.01\n",
      "37 550 -0.785460948944 -0.000112202587902 0.01\n",
      "valid_acc 75.76\n",
      "38 0 -0.782376229763 -0.000112202587902 0.01\n",
      "38 50 -0.637903392315 -0.000112202587902 0.01\n",
      "38 100 -0.767036616802 -0.000112202587902 0.01\n",
      "38 150 -0.501023113728 -0.000112202587902 0.01\n",
      "38 200 -0.620280206203 -0.000112202587902 0.01\n",
      "38 250 -0.965233147144 -0.000112202587902 0.01\n",
      "38 300 -0.742073357105 -0.000112202587902 0.01\n",
      "38 350 -0.626462340355 -0.000112202587902 0.01\n",
      "38 400 -0.95948690176 -0.000112202587902 0.01\n",
      "38 450 -0.718340158463 -0.000112202587902 0.01\n",
      "38 500 -0.754166781902 -0.000112202587902 0.01\n",
      "38 550 -0.658983707428 -0.000112202587902 0.01\n",
      "valid_acc 77.79166666666667\n",
      "39 0 -0.601301074028 -0.000112202587902 0.01\n",
      "39 50 -0.737544298172 -0.000112202587902 0.01\n",
      "39 100 -0.700616836548 -0.000112202587902 0.01\n",
      "39 150 -0.887045204639 -0.000112202587902 0.01\n",
      "39 200 -0.9743090868 -0.000112202587902 0.01\n",
      "39 250 -0.74027723074 -0.000112202587902 0.01\n",
      "39 300 -0.550084233284 -0.000112202587902 0.01\n",
      "39 350 -0.633387386799 -0.000112202587902 0.01\n",
      "39 400 -0.560686349869 -0.000112202587902 0.01\n",
      "39 450 -0.510309398174 -0.000112202587902 0.01\n",
      "39 500 -0.922400593758 -0.000112202587902 0.01\n",
      "39 550 -0.838894486427 -0.000112202587902 0.01\n",
      "valid_acc 77.35\n",
      "40 0 -0.604630947113 -0.000112202587902 0.01\n",
      "40 50 -0.80502820015 -0.000112202587902 0.01\n",
      "40 100 -0.723993003368 -0.000112202587902 0.01\n",
      "40 150 -0.914473593235 -0.000112202587902 0.01\n",
      "40 200 -0.841212153435 -0.000112202587902 0.01\n",
      "40 250 -0.958129584789 -0.000112202587902 0.01\n",
      "40 300 -0.667169094086 -0.000112202587902 0.01\n",
      "40 350 -0.899476289749 -0.000112202587902 0.01\n",
      "40 400 -0.811425089836 -0.000112202587902 0.01\n",
      "40 450 -1.17802906036 -0.000112202587902 0.01\n",
      "40 500 -0.838473200798 -0.000112202587902 0.01\n",
      "40 550 -0.839331746101 -0.000112202587902 0.01\n",
      "valid_acc 78.21666666666667\n",
      "41 0 -0.752184927464 -0.000112202587902 0.01\n",
      "41 50 -0.833226382732 -0.000112202587902 0.01\n",
      "41 100 -0.777242660522 -0.000112202587902 0.01\n",
      "41 150 -0.967317819595 -0.000112202587902 0.01\n",
      "41 200 -0.589810371399 -0.000112202587902 0.01\n",
      "41 250 -0.785087049007 -0.000112202587902 0.01\n",
      "41 300 -0.773467540741 -0.000112202587902 0.01\n",
      "41 350 -0.534242510796 -0.000112202587902 0.01\n",
      "41 400 -0.667821645737 -0.000112202587902 0.01\n",
      "41 450 -0.587174773216 -0.000112202587902 0.01\n",
      "41 500 -0.954535663128 -0.000112202587902 0.01\n",
      "41 550 -0.96244764328 -0.000112202587902 0.01\n",
      "valid_acc 76.89833333333334\n",
      "42 0 -0.894358158112 -0.000112202587902 0.01\n",
      "42 50 -0.679703116417 -0.000112202587902 0.01\n",
      "42 100 -0.8293222785 -0.000112202587902 0.01\n",
      "42 150 -0.799442827702 -0.000112202587902 0.01\n",
      "42 200 -0.978929340839 -0.000112202587902 0.01\n",
      "42 250 -0.715289890766 -0.000112202587902 0.01\n",
      "42 300 -1.0307956934 -0.000112202587902 0.01\n",
      "42 350 -0.813206851482 -0.000112202587902 0.01\n",
      "42 400 -1.09521007538 -0.000112202587902 0.01\n",
      "42 450 -0.827242136002 -0.000112202587902 0.01\n",
      "42 500 -0.834038317204 -0.000112202587902 0.01\n",
      "42 550 -0.690669298172 -0.000112202587902 0.01\n",
      "valid_acc 75.06\n",
      "43 0 -0.766748726368 -0.000112202587902 0.01\n",
      "43 50 -0.743621528149 -0.000112202587902 0.01\n",
      "43 100 -0.789323449135 -0.000112202587902 0.01\n",
      "43 150 -1.01729595661 -0.000112202587902 0.01\n",
      "43 200 -0.67360419035 -0.000112202587902 0.01\n",
      "43 250 -0.922679185867 -0.000112202587902 0.01\n",
      "43 300 -0.710066616535 -0.000112202587902 0.01\n",
      "43 350 -0.797106385231 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 400 -1.21473741531 -0.000112202587902 0.01\n",
      "43 450 -0.780671298504 -0.000112202587902 0.01\n",
      "43 500 -0.763305485249 -0.000112202587902 0.01\n",
      "43 550 -0.986675262451 -0.000112202587902 0.01\n",
      "valid_acc 72.72333333333333\n",
      "44 0 -0.560661792755 -0.000112202587902 0.01\n",
      "44 50 -0.841722548008 -0.000112202587902 0.01\n",
      "44 100 -0.659147262573 -0.000112202587902 0.01\n",
      "44 150 -0.841374158859 -0.000112202587902 0.01\n",
      "44 200 -1.29826307297 -0.000112202587902 0.01\n",
      "44 250 -0.892942726612 -0.000112202587902 0.01\n",
      "44 300 -0.700051546097 -0.000112202587902 0.01\n",
      "44 350 -0.966121375561 -0.000112202587902 0.01\n",
      "44 400 -0.716448187828 -0.000112202587902 0.01\n",
      "44 450 -1.39912796021 -0.000112202587902 0.01\n",
      "44 500 -0.946059465408 -0.000112202587902 0.01\n",
      "44 550 -0.655905842781 -0.000112202587902 0.01\n",
      "valid_acc 71.53500000000001\n",
      "45 0 -0.772949278355 -0.000112202587902 0.01\n",
      "45 50 -0.883020222187 -0.000112202587902 0.01\n",
      "45 100 -1.06702172756 -0.000112202587902 0.01\n",
      "45 150 -0.829590976238 -0.000112202587902 0.01\n",
      "45 200 -0.899008095264 -0.000112202587902 0.01\n",
      "45 250 -0.746445417404 -0.000112202587902 0.01\n",
      "45 300 -0.766833126545 -0.000112202587902 0.01\n",
      "45 350 -1.05002200603 -0.000112202587902 0.01\n",
      "45 400 -0.87357121706 -0.000112202587902 0.01\n",
      "45 450 -0.945506215096 -0.000112202587902 0.01\n",
      "45 500 -0.792427897453 -0.000112202587902 0.01\n",
      "45 550 -1.07907414436 -0.000112202587902 0.01\n",
      "valid_acc 74.98333333333333\n",
      "46 0 -0.7272824049 -0.000112202587902 0.01\n",
      "46 50 -1.00436031818 -0.000112202587902 0.01\n",
      "46 100 -0.693506598473 -0.000112202587902 0.01\n",
      "46 150 -0.9566655159 -0.000112202587902 0.01\n",
      "46 200 -0.633663415909 -0.000112202587902 0.01\n",
      "46 250 -0.86416977644 -0.000112202587902 0.01\n",
      "46 300 -0.640816867352 -0.000112202587902 0.01\n",
      "46 350 -0.662136554718 -0.000112202587902 0.01\n",
      "46 400 -0.612194895744 -0.000112202587902 0.01\n",
      "46 450 -0.702787220478 -0.000112202587902 0.01\n",
      "46 500 -1.05380916595 -0.000112202587902 0.01\n",
      "46 550 -0.648333907127 -0.000112202587902 0.01\n",
      "valid_acc 76.85333333333332\n",
      "47 0 -0.666262686253 -0.000112202587902 0.01\n",
      "47 50 -1.03118777275 -0.000112202587902 0.01\n",
      "47 100 -0.897649168968 -0.000112202587902 0.01\n",
      "47 150 -0.655808091164 -0.000112202587902 0.01\n",
      "47 200 -1.03338217735 -0.000112202587902 0.01\n",
      "47 250 -0.49954944849 -0.000112202587902 0.01\n",
      "47 300 -1.1604590416 -0.000112202587902 0.01\n",
      "47 350 -0.815763115883 -0.000112202587902 0.01\n",
      "47 400 -0.896906733513 -0.000112202587902 0.01\n",
      "47 450 -0.93252491951 -0.000112202587902 0.01\n",
      "47 500 -0.726517498493 -0.000112202587902 0.01\n",
      "47 550 -0.726301789284 -0.000112202587902 0.01\n",
      "valid_acc 76.14999999999999\n",
      "48 0 -1.12587809563 -0.000112202587902 0.01\n",
      "48 50 -0.901296854019 -0.000112202587902 0.01\n",
      "48 100 -0.700489640236 -0.000112202587902 0.01\n",
      "48 150 -0.861990332603 -0.000112202587902 0.01\n",
      "48 200 -0.869911968708 -0.000112202587902 0.01\n",
      "48 250 -0.768312990665 -0.000112202587902 0.01\n",
      "48 300 -0.788683533669 -0.000112202587902 0.01\n",
      "48 350 -0.614928424358 -0.000112202587902 0.01\n",
      "48 400 -0.848378896713 -0.000112202587902 0.01\n",
      "48 450 -0.970139980316 -0.000112202587902 0.01\n",
      "48 500 -0.745389699936 -0.000112202587902 0.01\n",
      "48 550 -0.802948355675 -0.000112202587902 0.01\n",
      "valid_acc 77.48333333333333\n",
      "49 0 -0.807204067707 -0.000112202587902 0.01\n",
      "49 50 -0.5926181674 -0.000112202587902 0.01\n",
      "49 100 -0.577438056469 -0.000112202587902 0.01\n",
      "49 150 -0.82837587595 -0.000112202587902 0.01\n",
      "49 200 -0.607106089592 -0.000112202587902 0.01\n",
      "49 250 -0.946474015713 -0.000112202587902 0.01\n",
      "49 300 -0.644412219524 -0.000112202587902 0.01\n",
      "49 350 -0.722852766514 -0.000112202587902 0.01\n",
      "49 400 -0.533721864223 -0.000112202587902 0.01\n",
      "49 450 -0.859082043171 -0.000112202587902 0.01\n",
      "49 500 -0.754909694195 -0.000112202587902 0.01\n",
      "49 550 -0.713592708111 -0.000112202587902 0.01\n",
      "valid_acc 75.39333333333333\n",
      "50 0 -0.496781527996 -0.000112202587902 0.01\n",
      "50 50 -0.575070142746 -0.000112202587902 0.01\n",
      "50 100 -0.746084809303 -0.000112202587902 0.01\n",
      "50 150 -0.616649150848 -0.000112202587902 0.01\n",
      "50 200 -0.865334630013 -0.000112202587902 0.01\n",
      "50 250 -0.513427138329 -0.000112202587902 0.01\n",
      "50 300 -0.847970128059 -0.000112202587902 0.01\n",
      "50 350 -0.695986807346 -0.000112202587902 0.01\n",
      "50 400 -0.761444091797 -0.000112202587902 0.01\n",
      "50 450 -0.45698055625 -0.000112202587902 0.01\n",
      "50 500 -0.499860465527 -0.000112202587902 0.01\n",
      "50 550 -0.717564404011 -0.000112202587902 0.01\n",
      "valid_acc 74.095\n",
      "51 0 -0.583441674709 -0.000112202587902 0.01\n",
      "51 50 -1.11849188805 -0.000112202587902 0.01\n",
      "51 100 -0.571453809738 -0.000112202587902 0.01\n",
      "51 150 -0.917515695095 -0.000112202587902 0.01\n",
      "51 200 -0.6389798522 -0.000112202587902 0.01\n",
      "51 250 -0.92483061552 -0.000112202587902 0.01\n",
      "51 300 -0.613454222679 -0.000112202587902 0.01\n",
      "51 350 -0.697228074074 -0.000112202587902 0.01\n",
      "51 400 -0.67453789711 -0.000112202587902 0.01\n",
      "51 450 -0.582725822926 -0.000112202587902 0.01\n",
      "51 500 -0.546845853329 -0.000112202587902 0.01\n",
      "51 550 -0.622687220573 -0.000112202587902 0.01\n",
      "valid_acc 75.73833333333333\n",
      "52 0 -0.732134103775 -0.000112202587902 0.01\n",
      "52 50 -0.747127592564 -0.000112202587902 0.01\n",
      "52 100 -0.832378864288 -0.000112202587902 0.01\n",
      "52 150 -0.734276890755 -0.000112202587902 0.01\n",
      "52 200 -0.715042889118 -0.000112202587902 0.01\n",
      "52 250 -1.08745574951 -0.000112202587902 0.01\n",
      "52 300 -0.810323774815 -0.000112202587902 0.01\n",
      "52 350 -0.622736990452 -0.000112202587902 0.01\n",
      "52 400 -0.479956448078 -0.000112202587902 0.01\n",
      "52 450 -0.806446135044 -0.000112202587902 0.01\n",
      "52 500 -0.468951046467 -0.000112202587902 0.01\n",
      "52 550 -0.515350699425 -0.000112202587902 0.01\n",
      "valid_acc 78.36166666666666\n",
      "53 0 -0.828772306442 -0.000112202587902 0.01\n",
      "53 50 -0.563708126545 -0.000112202587902 0.01\n",
      "53 100 -1.0213341713 -0.000112202587902 0.01\n",
      "53 150 -0.604158878326 -0.000112202587902 0.01\n",
      "53 200 -0.752194166183 -0.000112202587902 0.01\n",
      "53 250 -0.714362323284 -0.000112202587902 0.01\n",
      "53 300 -0.689544379711 -0.000112202587902 0.01\n",
      "53 350 -0.739537715912 -0.000112202587902 0.01\n",
      "53 400 -0.667872667313 -0.000112202587902 0.01\n",
      "53 450 -0.851990759373 -0.000112202587902 0.01\n",
      "53 500 -0.527604877949 -0.000112202587902 0.01\n",
      "53 550 -0.701576709747 -0.000112202587902 0.01\n",
      "valid_acc 78.025\n",
      "54 0 -0.751285851002 -0.000112202587902 0.01\n",
      "54 50 -0.822109937668 -0.000112202587902 0.01\n",
      "54 100 -0.819692134857 -0.000112202587902 0.01\n",
      "54 150 -0.781393885612 -0.000112202587902 0.01\n",
      "54 200 -0.599238336086 -0.000112202587902 0.01\n",
      "54 250 -0.959752678871 -0.000112202587902 0.01\n",
      "54 300 -0.910968840122 -0.000112202587902 0.01\n",
      "54 350 -0.639237463474 -0.000112202587902 0.01\n",
      "54 400 -0.475719302893 -0.000112202587902 0.01\n",
      "54 450 -0.949222207069 -0.000112202587902 0.01\n",
      "54 500 -0.806299209595 -0.000112202587902 0.01\n",
      "54 550 -0.472577691078 -0.000112202587902 0.01\n",
      "valid_acc 78.93833333333333\n",
      "55 0 -0.829360663891 -0.000112202587902 0.01\n",
      "55 50 -0.600332021713 -0.000112202587902 0.01\n",
      "55 100 -0.90046364069 -0.000112202587902 0.01\n",
      "55 150 -0.925664722919 -0.000112202587902 0.01\n",
      "55 200 -0.887863397598 -0.000112202587902 0.01\n",
      "55 250 -0.590217411518 -0.000112202587902 0.01\n",
      "55 300 -0.829982817173 -0.000112202587902 0.01\n",
      "55 350 -0.711695194244 -0.000112202587902 0.01\n",
      "55 400 -0.746603012085 -0.000112202587902 0.01\n",
      "55 450 -0.852025747299 -0.000112202587902 0.01\n",
      "55 500 -0.511145293713 -0.000112202587902 0.01\n",
      "55 550 -0.689260542393 -0.000112202587902 0.01\n",
      "valid_acc 80.33500000000001\n",
      "best valid_acc 80.33500000000001\n",
      "56 0 -0.731139063835 -0.000112202587902 0.01\n",
      "56 50 -0.950915575027 -0.000112202587902 0.01\n",
      "56 100 -0.801579415798 -0.000112202587902 0.01\n",
      "56 150 -0.731820821762 -0.000112202587902 0.01\n",
      "56 200 -1.14309406281 -0.000112202587902 0.01\n",
      "56 250 -0.456720620394 -0.000112202587902 0.01\n",
      "56 300 -0.893048882484 -0.000112202587902 0.01\n",
      "56 350 -0.611996173859 -0.000112202587902 0.01\n",
      "56 400 -0.581286072731 -0.000112202587902 0.01\n",
      "56 450 -0.550316989422 -0.000112202587902 0.01\n",
      "56 500 -0.742323935032 -0.000112202587902 0.01\n",
      "56 550 -0.873440623283 -0.000112202587902 0.01\n",
      "valid_acc 79.88666666666666\n",
      "57 0 -0.790513157845 -0.000112202587902 0.01\n",
      "57 50 -0.477443814278 -0.000112202587902 0.01\n",
      "57 100 -0.86554902792 -0.000112202587902 0.01\n",
      "57 150 -0.493998229504 -0.000112202587902 0.01\n",
      "57 200 -0.857462763786 -0.000112202587902 0.01\n",
      "57 250 -0.472088694572 -0.000112202587902 0.01\n",
      "57 300 -0.86275178194 -0.000112202587902 0.01\n",
      "57 350 -0.575243294239 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57 400 -0.482666105032 -0.000112202587902 0.01\n",
      "57 450 -0.543205857277 -0.000112202587902 0.01\n",
      "57 500 -0.713469982147 -0.000112202587902 0.01\n",
      "57 550 -0.559324324131 -0.000112202587902 0.01\n",
      "valid_acc 76.17833333333334\n",
      "58 0 -0.470753580332 -0.000112202587902 0.01\n",
      "58 50 -0.565346837044 -0.000112202587902 0.01\n",
      "58 100 -0.499830126762 -0.000112202587902 0.01\n",
      "58 150 -0.448119670153 -0.000112202587902 0.01\n",
      "58 200 -0.893559336662 -0.000112202587902 0.01\n",
      "58 250 -0.811080694199 -0.000112202587902 0.01\n",
      "58 300 -0.904327690601 -0.000112202587902 0.01\n",
      "58 350 -0.652942955494 -0.000112202587902 0.01\n",
      "58 400 -0.729658067226 -0.000112202587902 0.01\n",
      "58 450 -0.466366350651 -0.000112202587902 0.01\n",
      "58 500 -0.338601529598 -0.000112202587902 0.01\n",
      "58 550 -0.743045628071 -0.000112202587902 0.01\n",
      "valid_acc 78.50333333333333\n",
      "59 0 -0.585690975189 -0.000112202587902 0.01\n",
      "59 50 -0.666297316551 -0.000112202587902 0.01\n",
      "59 100 -0.636935472488 -0.000112202587902 0.01\n",
      "59 150 -0.767390727997 -0.000112202587902 0.01\n",
      "59 200 -0.778753757477 -0.000112202587902 0.01\n",
      "59 250 -0.637694180012 -0.000112202587902 0.01\n",
      "59 300 -0.563474655151 -0.000112202587902 0.01\n",
      "59 350 -0.997043609619 -0.000112202587902 0.01\n",
      "59 400 -0.954598605633 -0.000112202587902 0.01\n",
      "59 450 -0.744355380535 -0.000112202587902 0.01\n",
      "59 500 -0.510385751724 -0.000112202587902 0.01\n",
      "59 550 -0.480589747429 -0.000112202587902 0.01\n",
      "valid_acc 80.645\n",
      "best valid_acc 80.645\n",
      "60 0 -0.557274878025 -0.000112202587902 0.01\n",
      "60 50 -0.845082640648 -0.000112202587902 0.01\n",
      "60 100 -1.13286578655 -0.000112202587902 0.01\n",
      "60 150 -0.812770664692 -0.000112202587902 0.01\n",
      "60 200 -0.786364734173 -0.000112202587902 0.01\n",
      "60 250 -0.793171167374 -0.000112202587902 0.01\n",
      "60 300 -0.488040059805 -0.000112202587902 0.01\n",
      "60 350 -0.513128876686 -0.000112202587902 0.01\n",
      "60 400 -0.569474816322 -0.000112202587902 0.01\n",
      "60 450 -0.984847307205 -0.000112202587902 0.01\n",
      "60 500 -1.09519946575 -0.000112202587902 0.01\n",
      "60 550 -0.642833888531 -0.000112202587902 0.01\n",
      "valid_acc 77.59166666666667\n",
      "61 0 -0.588406383991 -0.000112202587902 0.01\n",
      "61 50 -0.485206782818 -0.000112202587902 0.01\n",
      "61 100 -0.570270061493 -0.000112202587902 0.01\n",
      "61 150 -0.604605734348 -0.000112202587902 0.01\n",
      "61 200 -0.500403165817 -0.000112202587902 0.01\n",
      "61 250 -0.618351459503 -0.000112202587902 0.01\n",
      "61 300 -0.533682703972 -0.000112202587902 0.01\n",
      "61 350 -0.674032151699 -0.000112202587902 0.01\n",
      "61 400 -0.83864325285 -0.000112202587902 0.01\n",
      "61 450 -0.651875972748 -0.000112202587902 0.01\n",
      "61 500 -1.1895737648 -0.000112202587902 0.01\n",
      "61 550 -0.523512601852 -0.000112202587902 0.01\n",
      "valid_acc 80.84166666666667\n",
      "best valid_acc 80.84166666666667\n",
      "62 0 -0.706240713596 -0.000112202587902 0.01\n",
      "62 50 -0.813365340233 -0.000112202587902 0.01\n",
      "62 100 -0.567342281342 -0.000112202587902 0.01\n",
      "62 150 -0.757938563824 -0.000112202587902 0.01\n",
      "62 200 -0.862844109535 -0.000112202587902 0.01\n",
      "62 250 -0.584889233112 -0.000112202587902 0.01\n",
      "62 300 -0.826302349567 -0.000112202587902 0.01\n",
      "62 350 -0.667921602726 -0.000112202587902 0.01\n",
      "62 400 -0.484110176563 -0.000112202587902 0.01\n",
      "62 450 -0.687647104263 -0.000112202587902 0.01\n",
      "62 500 -0.734355151653 -0.000112202587902 0.01\n",
      "62 550 -0.620041787624 -0.000112202587902 0.01\n",
      "valid_acc 81.18166666666666\n",
      "best valid_acc 81.18166666666666\n",
      "63 0 -0.614448070526 -0.000112202587902 0.01\n",
      "63 50 -1.0329272747 -0.000112202587902 0.01\n",
      "63 100 -0.604020774364 -0.000112202587902 0.01\n",
      "63 150 -0.72216385603 -0.000112202587902 0.01\n",
      "63 200 -0.689642250538 -0.000112202587902 0.01\n",
      "63 250 -0.601412177086 -0.000112202587902 0.01\n",
      "63 300 -0.514044821262 -0.000112202587902 0.01\n",
      "63 350 -0.594485461712 -0.000112202587902 0.01\n",
      "63 400 -0.60677921772 -0.000112202587902 0.01\n",
      "63 450 -0.797562479973 -0.000112202587902 0.01\n",
      "63 500 -0.848721921444 -0.000112202587902 0.01\n",
      "63 550 -0.703183114529 -0.000112202587902 0.01\n",
      "valid_acc 80.31333333333333\n",
      "64 0 -0.550311148167 -0.000112202587902 0.01\n",
      "64 50 -0.741494297981 -0.000112202587902 0.01\n",
      "64 100 -0.5987880826 -0.000112202587902 0.01\n",
      "64 150 -0.581367313862 -0.000112202587902 0.01\n",
      "64 200 -0.722076058388 -0.000112202587902 0.01\n",
      "64 250 -0.64568066597 -0.000112202587902 0.01\n",
      "64 300 -0.5389970541 -0.000112202587902 0.01\n",
      "64 350 -0.754537940025 -0.000112202587902 0.01\n",
      "64 400 -0.647888720036 -0.000112202587902 0.01\n",
      "64 450 -0.611397087574 -0.000112202587902 0.01\n",
      "64 500 -0.563212931156 -0.000112202587902 0.01\n",
      "64 550 -0.686201810837 -0.000112202587902 0.01\n",
      "valid_acc 79.19333333333334\n",
      "65 0 -0.831326246262 -0.000112202587902 0.01\n",
      "65 50 -0.591747879982 -0.000112202587902 0.01\n",
      "65 100 -0.645516455173 -0.000112202587902 0.01\n",
      "65 150 -0.552339196205 -0.000112202587902 0.01\n",
      "65 200 -0.687091350555 -0.000112202587902 0.01\n",
      "65 250 -0.536503314972 -0.000112202587902 0.01\n",
      "65 300 -0.865339457989 -0.000112202587902 0.01\n",
      "65 350 -0.792307496071 -0.000112202587902 0.01\n",
      "65 400 -0.693919360638 -0.000112202587902 0.01\n",
      "65 450 -0.978867948055 -0.000112202587902 0.01\n",
      "65 500 -0.563427209854 -0.000112202587902 0.01\n",
      "65 550 -0.840477347374 -0.000112202587902 0.01\n",
      "valid_acc 79.845\n",
      "66 0 -0.570008993149 -0.000112202587902 0.01\n",
      "66 50 -0.464550048113 -0.000112202587902 0.01\n",
      "66 100 -0.481311202049 -0.000112202587902 0.01\n",
      "66 150 -0.594287753105 -0.000112202587902 0.01\n",
      "66 200 -0.549759209156 -0.000112202587902 0.01\n",
      "66 250 -0.766834795475 -0.000112202587902 0.01\n",
      "66 300 -0.672146558762 -0.000112202587902 0.01\n",
      "66 350 -0.490667104721 -0.000112202587902 0.01\n",
      "66 400 -0.620205402374 -0.000112202587902 0.01\n",
      "66 450 -0.582179307938 -0.000112202587902 0.01\n",
      "66 500 -0.644592642784 -0.000112202587902 0.01\n",
      "66 550 -0.459029167891 -0.000112202587902 0.01\n",
      "valid_acc 78.16499999999999\n",
      "67 0 -0.753557682037 -0.000112202587902 0.01\n",
      "67 50 -0.684627473354 -0.000112202587902 0.01\n",
      "67 100 -0.821030378342 -0.000112202587902 0.01\n",
      "67 150 -0.558816611767 -0.000112202587902 0.01\n",
      "67 200 -0.764763772488 -0.000112202587902 0.01\n",
      "67 250 -0.348223805428 -0.000112202587902 0.01\n",
      "67 300 -0.507696688175 -0.000112202587902 0.01\n",
      "67 350 -0.835712254047 -0.000112202587902 0.01\n",
      "67 400 -0.812129676342 -0.000112202587902 0.01\n",
      "67 450 -0.552847445011 -0.000112202587902 0.01\n",
      "67 500 -0.352099150419 -0.000112202587902 0.01\n",
      "67 550 -0.889608204365 -0.000112202587902 0.01\n",
      "valid_acc 77.53166666666667\n",
      "68 0 -0.61180961132 -0.000112202587902 0.01\n",
      "68 50 -0.646708250046 -0.000112202587902 0.01\n",
      "68 100 -0.737657368183 -0.000112202587902 0.01\n",
      "68 150 -0.494717180729 -0.000112202587902 0.01\n",
      "68 200 -0.499888569117 -0.000112202587902 0.01\n",
      "68 250 -0.519946396351 -0.000112202587902 0.01\n",
      "68 300 -0.61552298069 -0.000112202587902 0.01\n",
      "68 350 -0.483822435141 -0.000112202587902 0.01\n",
      "68 400 -0.608637452126 -0.000112202587902 0.01\n",
      "68 450 -0.491749644279 -0.000112202587902 0.01\n",
      "68 500 -0.722153186798 -0.000112202587902 0.01\n",
      "68 550 -0.453147232533 -0.000112202587902 0.01\n",
      "valid_acc 80.45833333333333\n",
      "69 0 -0.562183022499 -0.000112202587902 0.01\n",
      "69 50 -1.05187559128 -0.000112202587902 0.01\n",
      "69 100 -0.579371273518 -0.000112202587902 0.01\n",
      "69 150 -0.846705853939 -0.000112202587902 0.01\n",
      "69 200 -0.699180603027 -0.000112202587902 0.01\n",
      "69 250 -0.549706459045 -0.000112202587902 0.01\n",
      "69 300 -0.775961756706 -0.000112202587902 0.01\n",
      "69 350 -0.44484680891 -0.000112202587902 0.01\n",
      "69 400 -0.729874134064 -0.000112202587902 0.01\n",
      "69 450 -0.50829988718 -0.000112202587902 0.01\n",
      "69 500 -0.884213805199 -0.000112202587902 0.01\n",
      "69 550 -0.493721276522 -0.000112202587902 0.01\n",
      "valid_acc 78.45\n",
      "70 0 -0.525245666504 -0.000112202587902 0.01\n",
      "70 50 -0.711679279804 -0.000112202587902 0.01\n",
      "70 100 -0.679878532887 -0.000112202587902 0.01\n",
      "70 150 -0.627071321011 -0.000112202587902 0.01\n",
      "70 200 -0.41970166564 -0.000112202587902 0.01\n",
      "70 250 -0.760175168514 -0.000112202587902 0.01\n",
      "70 300 -0.733814716339 -0.000112202587902 0.01\n",
      "70 350 -0.623445570469 -0.000112202587902 0.01\n",
      "70 400 -0.47972714901 -0.000112202587902 0.01\n",
      "70 450 -0.613128781319 -0.000112202587902 0.01\n",
      "70 500 -0.715051054955 -0.000112202587902 0.01\n",
      "70 550 -0.866258919239 -0.000112202587902 0.01\n",
      "valid_acc 80.42666666666666\n",
      "71 0 -0.663254678249 -0.000112202587902 0.01\n",
      "71 50 -0.76100730896 -0.000112202587902 0.01\n",
      "71 100 -0.877789855003 -0.000112202587902 0.01\n",
      "71 150 -0.611882507801 -0.000112202587902 0.01\n",
      "71 200 -0.821135401726 -0.000112202587902 0.01\n",
      "71 250 -0.739791929722 -0.000112202587902 0.01\n",
      "71 300 -0.638088107109 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71 350 -0.663123011589 -0.000112202587902 0.01\n",
      "71 400 -0.534290254116 -0.000112202587902 0.01\n",
      "71 450 -0.508787095547 -0.000112202587902 0.01\n",
      "71 500 -0.356436192989 -0.000112202587902 0.01\n",
      "71 550 -0.658250808716 -0.000112202587902 0.01\n",
      "valid_acc 78.10833333333333\n",
      "72 0 -0.673221111298 -0.000112202587902 0.01\n",
      "72 50 -0.763592064381 -0.000112202587902 0.01\n",
      "72 100 -0.673849403858 -0.000112202587902 0.01\n",
      "72 150 -0.785580515862 -0.000112202587902 0.01\n",
      "72 200 -0.574935972691 -0.000112202587902 0.01\n",
      "72 250 -0.594264626503 -0.000112202587902 0.01\n",
      "72 300 -0.522734344006 -0.000112202587902 0.01\n",
      "72 350 -0.894905686378 -0.000112202587902 0.01\n",
      "72 400 -0.500092506409 -0.000112202587902 0.01\n",
      "72 450 -0.565209388733 -0.000112202587902 0.01\n",
      "72 500 -0.907502949238 -0.000112202587902 0.01\n",
      "72 550 -0.664677679539 -0.000112202587902 0.01\n",
      "valid_acc 78.645\n",
      "73 0 -0.729112863541 -0.000112202587902 0.01\n",
      "73 50 -0.5487383008 -0.000112202587902 0.01\n",
      "73 100 -0.452591091394 -0.000112202587902 0.01\n",
      "73 150 -0.723404943943 -0.000112202587902 0.01\n",
      "73 200 -0.479881137609 -0.000112202587902 0.01\n",
      "73 250 -0.608432590961 -0.000112202587902 0.01\n",
      "73 300 -0.933769226074 -0.000112202587902 0.01\n",
      "73 350 -0.920958280563 -0.000112202587902 0.01\n",
      "73 400 -0.430601805449 -0.000112202587902 0.01\n",
      "73 450 -0.775568068027 -0.000112202587902 0.01\n",
      "73 500 -0.560193955898 -0.000112202587902 0.01\n",
      "73 550 -0.604109525681 -0.000112202587902 0.01\n",
      "valid_acc 78.58333333333334\n",
      "74 0 -0.528481960297 -0.000112202587902 0.01\n",
      "74 50 -0.581626951694 -0.000112202587902 0.01\n",
      "74 100 -0.747179567814 -0.000112202587902 0.01\n",
      "74 150 -1.07773089409 -0.000112202587902 0.01\n",
      "74 200 -0.589391171932 -0.000112202587902 0.01\n",
      "74 250 -1.16997087002 -0.000112202587902 0.01\n",
      "74 300 -0.859362483025 -0.000112202587902 0.01\n",
      "74 350 -0.731021642685 -0.000112202587902 0.01\n",
      "74 400 -0.769325733185 -0.000112202587902 0.01\n",
      "74 450 -0.501139640808 -0.000112202587902 0.01\n",
      "74 500 -0.646934330463 -0.000112202587902 0.01\n",
      "74 550 -0.726143598557 -0.000112202587902 0.01\n",
      "valid_acc 79.30666666666667\n",
      "75 0 -0.607834279537 -0.000112202587902 0.01\n",
      "75 50 -0.484270364046 -0.000112202587902 0.01\n",
      "75 100 -0.777851164341 -0.000112202587902 0.01\n",
      "75 150 -0.79471808672 -0.000112202587902 0.01\n",
      "75 200 -0.758248746395 -0.000112202587902 0.01\n",
      "75 250 -1.02721583843 -0.000112202587902 0.01\n",
      "75 300 -0.694262385368 -0.000112202587902 0.01\n",
      "75 350 -0.390720456839 -0.000112202587902 0.01\n",
      "75 400 -0.655848562717 -0.000112202587902 0.01\n",
      "75 450 -0.602988600731 -0.000112202587902 0.01\n",
      "75 500 -0.846168220043 -0.000112202587902 0.01\n",
      "75 550 -1.07296001911 -0.000112202587902 0.01\n",
      "valid_acc 74.36833333333334\n",
      "76 0 -0.540608882904 -0.000112202587902 0.01\n",
      "76 50 -0.584235966206 -0.000112202587902 0.01\n",
      "76 100 -0.712187290192 -0.000112202587902 0.01\n",
      "76 150 -0.683321833611 -0.000112202587902 0.01\n",
      "76 200 -0.863071501255 -0.000112202587902 0.01\n",
      "76 250 -0.445641160011 -0.000112202587902 0.01\n",
      "76 300 -0.978717327118 -0.000112202587902 0.01\n",
      "76 350 -0.541950941086 -0.000112202587902 0.01\n",
      "76 400 -0.363027483225 -0.000112202587902 0.01\n",
      "76 450 -0.491010218859 -0.000112202587902 0.01\n",
      "76 500 -0.804258823395 -0.000112202587902 0.01\n",
      "76 550 -0.556309878826 -0.000112202587902 0.01\n",
      "valid_acc 80.89\n",
      "77 0 -0.702097833157 -0.000112202587902 0.01\n",
      "77 50 -1.02161073685 -0.000112202587902 0.01\n",
      "77 100 -0.79550665617 -0.000112202587902 0.01\n",
      "77 150 -0.632730901241 -0.000112202587902 0.01\n",
      "77 200 -0.702944636345 -0.000112202587902 0.01\n",
      "77 250 -0.771452188492 -0.000112202587902 0.01\n",
      "77 300 -0.541844785213 -0.000112202587902 0.01\n",
      "77 350 -0.563979148865 -0.000112202587902 0.01\n",
      "77 400 -0.44492316246 -0.000112202587902 0.01\n",
      "77 450 -0.633125424385 -0.000112202587902 0.01\n",
      "77 500 -0.795518100262 -0.000112202587902 0.01\n",
      "77 550 -0.929859220982 -0.000112202587902 0.01\n",
      "valid_acc 77.28166666666667\n",
      "78 0 -0.754711091518 -0.000112202587902 0.01\n",
      "78 50 -0.649134576321 -0.000112202587902 0.01\n",
      "78 100 -0.636634409428 -0.000112202587902 0.01\n",
      "78 150 -0.517560184002 -0.000112202587902 0.01\n",
      "78 200 -0.927013218403 -0.000112202587902 0.01\n",
      "78 250 -0.659576654434 -0.000112202587902 0.01\n",
      "78 300 -0.964405298233 -0.000112202587902 0.01\n",
      "78 350 -0.622018754482 -0.000112202587902 0.01\n",
      "78 400 -0.535965502262 -0.000112202587902 0.01\n",
      "78 450 -0.587013602257 -0.000112202587902 0.01\n",
      "78 500 -0.423459142447 -0.000112202587902 0.01\n",
      "78 550 -0.463441312313 -0.000112202587902 0.01\n",
      "valid_acc 80.045\n",
      "79 0 -0.585850119591 -0.000112202587902 0.01\n",
      "79 50 -0.803311288357 -0.000112202587902 0.01\n",
      "79 100 -0.546316087246 -0.000112202587902 0.01\n",
      "79 150 -0.865651547909 -0.000112202587902 0.01\n",
      "79 200 -0.538493335247 -0.000112202587902 0.01\n",
      "79 250 -0.500631511211 -0.000112202587902 0.01\n",
      "79 300 -0.309778571129 -0.000112202587902 0.01\n",
      "79 350 -0.604791402817 -0.000112202587902 0.01\n",
      "79 400 -0.965327143669 -0.000112202587902 0.01\n",
      "79 450 -0.870816290379 -0.000112202587902 0.01\n",
      "79 500 -0.71417504549 -0.000112202587902 0.01\n",
      "79 550 -0.609602153301 -0.000112202587902 0.01\n",
      "valid_acc 78.11166666666666\n",
      "80 0 -0.833780825138 -0.000112202587902 0.01\n",
      "80 50 -0.667421519756 -0.000112202587902 0.01\n",
      "80 100 -0.499669879675 -0.000112202587902 0.01\n",
      "80 150 -0.428962677717 -0.000112202587902 0.01\n",
      "80 200 -0.953308701515 -0.000112202587902 0.01\n",
      "80 250 -0.571823894978 -0.000112202587902 0.01\n",
      "80 300 -0.393024742603 -0.000112202587902 0.01\n",
      "80 350 -0.714814960957 -0.000112202587902 0.01\n",
      "80 400 -0.633146762848 -0.000112202587902 0.01\n",
      "80 450 -0.640738904476 -0.000112202587902 0.01\n",
      "80 500 -0.68417429924 -0.000112202587902 0.01\n",
      "80 550 -0.610380768776 -0.000112202587902 0.01\n",
      "valid_acc 79.58\n",
      "81 0 -0.670133054256 -0.000112202587902 0.01\n",
      "81 50 -0.785099029541 -0.000112202587902 0.01\n",
      "81 100 -0.799298763275 -0.000112202587902 0.01\n",
      "81 150 -0.842777967453 -0.000112202587902 0.01\n",
      "81 200 -0.973524093628 -0.000112202587902 0.01\n",
      "81 250 -0.50888890028 -0.000112202587902 0.01\n",
      "81 300 -0.718113839626 -0.000112202587902 0.01\n",
      "81 350 -0.490122526884 -0.000112202587902 0.01\n",
      "81 400 -0.692939758301 -0.000112202587902 0.01\n",
      "81 450 -0.696554481983 -0.000112202587902 0.01\n",
      "81 500 -0.71622377634 -0.000112202587902 0.01\n",
      "81 550 -0.409277319908 -0.000112202587902 0.01\n",
      "valid_acc 80.28999999999999\n",
      "82 0 -1.16632044315 -0.000112202587902 0.01\n",
      "82 50 -1.09210705757 -0.000112202587902 0.01\n",
      "82 100 -0.781825423241 -0.000112202587902 0.01\n",
      "82 150 -0.922896802425 -0.000112202587902 0.01\n",
      "82 200 -0.833504736423 -0.000112202587902 0.01\n",
      "82 250 -0.838387668133 -0.000112202587902 0.01\n",
      "82 300 -0.514927685261 -0.000112202587902 0.01\n",
      "82 350 -0.735976159573 -0.000112202587902 0.01\n",
      "82 400 -0.712923049927 -0.000112202587902 0.01\n",
      "82 450 -0.679499387741 -0.000112202587902 0.01\n",
      "82 500 -0.587962210178 -0.000112202587902 0.01\n",
      "82 550 -0.653632640839 -0.000112202587902 0.01\n",
      "valid_acc 78.53999999999999\n",
      "83 0 -0.764808118343 -0.000112202587902 0.01\n",
      "83 50 -0.828167796135 -0.000112202587902 0.01\n",
      "83 100 -0.943429946899 -0.000112202587902 0.01\n",
      "83 150 -0.586651980877 -0.000112202587902 0.01\n",
      "83 200 -0.877230226994 -0.000112202587902 0.01\n",
      "83 250 -1.14733684063 -0.000112202587902 0.01\n",
      "83 300 -0.472924351692 -0.000112202587902 0.01\n",
      "83 350 -0.933752596378 -0.000112202587902 0.01\n",
      "83 400 -0.554349184036 -0.000112202587902 0.01\n",
      "83 450 -0.631920218468 -0.000112202587902 0.01\n",
      "83 500 -0.638029456139 -0.000112202587902 0.01\n",
      "83 550 -0.91505074501 -0.000112202587902 0.01\n",
      "valid_acc 78.10166666666667\n",
      "84 0 -0.905136823654 -0.000112202587902 0.01\n",
      "84 50 -0.677885234356 -0.000112202587902 0.01\n",
      "84 100 -0.733975172043 -0.000112202587902 0.01\n",
      "84 150 -0.971509099007 -0.000112202587902 0.01\n",
      "84 200 -0.880803465843 -0.000112202587902 0.01\n",
      "84 250 -0.811674594879 -0.000112202587902 0.01\n",
      "84 300 -0.711901724339 -0.000112202587902 0.01\n",
      "84 350 -0.594425201416 -0.000112202587902 0.01\n",
      "84 400 -0.494998782873 -0.000112202587902 0.01\n",
      "84 450 -0.561698257923 -0.000112202587902 0.01\n",
      "84 500 -0.54413163662 -0.000112202587902 0.01\n",
      "84 550 -0.657520890236 -0.000112202587902 0.01\n",
      "valid_acc 79.485\n",
      "85 0 -0.600917875767 -0.000112202587902 0.01\n",
      "85 50 -0.674822926521 -0.000112202587902 0.01\n",
      "85 100 -1.08222615719 -0.000112202587902 0.01\n",
      "85 150 -0.609044253826 -0.000112202587902 0.01\n",
      "85 200 -0.791786551476 -0.000112202587902 0.01\n",
      "85 250 -0.841832876205 -0.000112202587902 0.01\n",
      "85 300 -0.5563326478 -0.000112202587902 0.01\n",
      "85 350 -0.784590542316 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85 400 -0.446919739246 -0.000112202587902 0.01\n",
      "85 450 -0.462840855122 -0.000112202587902 0.01\n",
      "85 500 -0.484367609024 -0.000112202587902 0.01\n",
      "85 550 -0.736560344696 -0.000112202587902 0.01\n",
      "valid_acc 79.72166666666666\n",
      "86 0 -0.982348918915 -0.000112202587902 0.01\n",
      "86 50 -1.00114500523 -0.000112202587902 0.01\n",
      "86 100 -0.709446430206 -0.000112202587902 0.01\n",
      "86 150 -0.61432993412 -0.000112202587902 0.01\n",
      "86 200 -0.591121137142 -0.000112202587902 0.01\n",
      "86 250 -0.626484453678 -0.000112202587902 0.01\n",
      "86 300 -0.461392968893 -0.000112202587902 0.01\n",
      "86 350 -0.669681549072 -0.000112202587902 0.01\n",
      "86 400 -0.714386165142 -0.000112202587902 0.01\n",
      "86 450 -0.547152280807 -0.000112202587902 0.01\n",
      "86 500 -0.493396878242 -0.000112202587902 0.01\n",
      "86 550 -0.49549856782 -0.000112202587902 0.01\n",
      "valid_acc 81.14666666666666\n",
      "87 0 -0.765102684498 -0.000112202587902 0.01\n",
      "87 50 -0.649802863598 -0.000112202587902 0.01\n",
      "87 100 -0.493044674397 -0.000112202587902 0.01\n",
      "87 150 -0.681083500385 -0.000112202587902 0.01\n",
      "87 200 -0.794640481472 -0.000112202587902 0.01\n",
      "87 250 -0.720964491367 -0.000112202587902 0.01\n",
      "87 300 -0.824047327042 -0.000112202587902 0.01\n",
      "87 350 -0.524540424347 -0.000112202587902 0.01\n",
      "87 400 -0.607931375504 -0.000112202587902 0.01\n",
      "87 450 -0.461046516895 -0.000112202587902 0.01\n",
      "87 500 -0.783969283104 -0.000112202587902 0.01\n",
      "87 550 -0.62584489584 -0.000112202587902 0.01\n",
      "valid_acc 79.59666666666666\n",
      "88 0 -0.93175560236 -0.000112202587902 0.01\n",
      "88 50 -0.620124995708 -0.000112202587902 0.01\n",
      "88 100 -0.778468310833 -0.000112202587902 0.01\n",
      "88 150 -0.674371421337 -0.000112202587902 0.01\n",
      "88 200 -0.520252764225 -0.000112202587902 0.01\n",
      "88 250 -0.575633943081 -0.000112202587902 0.01\n",
      "88 300 -0.630035758018 -0.000112202587902 0.01\n",
      "88 350 -0.618870735168 -0.000112202587902 0.01\n",
      "88 400 -0.587666332722 -0.000112202587902 0.01\n",
      "88 450 -0.99830031395 -0.000112202587902 0.01\n",
      "88 500 -0.553679585457 -0.000112202587902 0.01\n",
      "88 550 -0.536929666996 -0.000112202587902 0.01\n",
      "valid_acc 81.40666666666667\n",
      "best valid_acc 81.40666666666667\n",
      "89 0 -0.607777774334 -0.000112202587902 0.01\n",
      "89 50 -0.797837495804 -0.000112202587902 0.01\n",
      "89 100 -0.857706010342 -0.000112202587902 0.01\n",
      "89 150 -0.469825178385 -0.000112202587902 0.01\n",
      "89 200 -0.781408250332 -0.000112202587902 0.01\n",
      "89 250 -0.424388051033 -0.000112202587902 0.01\n",
      "89 300 -0.904208004475 -0.000112202587902 0.01\n",
      "89 350 -0.839239835739 -0.000112202587902 0.01\n",
      "89 400 -0.986044466496 -0.000112202587902 0.01\n",
      "89 450 -0.528661727905 -0.000112202587902 0.01\n",
      "89 500 -0.776296257973 -0.000112202587902 0.01\n",
      "89 550 -0.68423730135 -0.000112202587902 0.01\n",
      "valid_acc 79.94666666666667\n",
      "90 0 -0.655417561531 -0.000112202587902 0.01\n",
      "90 50 -0.443871080875 -0.000112202587902 0.01\n",
      "90 100 -0.916770100594 -0.000112202587902 0.01\n",
      "90 150 -0.84838783741 -0.000112202587902 0.01\n",
      "90 200 -0.327486485243 -0.000112202587902 0.01\n",
      "90 250 -0.931833326817 -0.000112202587902 0.01\n",
      "90 300 -0.554437994957 -0.000112202587902 0.01\n",
      "90 350 -0.903427302837 -0.000112202587902 0.01\n",
      "90 400 -0.550981342793 -0.000112202587902 0.01\n",
      "90 450 -0.542372941971 -0.000112202587902 0.01\n",
      "90 500 -0.777925312519 -0.000112202587902 0.01\n",
      "90 550 -0.506290793419 -0.000112202587902 0.01\n",
      "valid_acc 78.45166666666667\n",
      "91 0 -0.324305534363 -0.000112202587902 0.01\n",
      "91 50 -0.633727312088 -0.000112202587902 0.01\n",
      "91 100 -0.376020818949 -0.000112202587902 0.01\n",
      "91 150 -0.540376663208 -0.000112202587902 0.01\n",
      "91 200 -0.675852000713 -0.000112202587902 0.01\n",
      "91 250 -0.562407076359 -0.000112202587902 0.01\n",
      "91 300 -0.469884335995 -0.000112202587902 0.01\n",
      "91 350 -0.520574748516 -0.000112202587902 0.01\n",
      "91 400 -0.707256317139 -0.000112202587902 0.01\n",
      "91 450 -0.788675665855 -0.000112202587902 0.01\n",
      "91 500 -0.997620820999 -0.000112202587902 0.01\n",
      "91 550 -0.695210039616 -0.000112202587902 0.01\n",
      "valid_acc 81.085\n",
      "92 0 -0.440164148808 -0.000112202587902 0.01\n",
      "92 50 -0.70960599184 -0.000112202587902 0.01\n",
      "92 100 -0.455692648888 -0.000112202587902 0.01\n",
      "92 150 -0.515543758869 -0.000112202587902 0.01\n",
      "92 200 -0.880060911179 -0.000112202587902 0.01\n",
      "92 250 -0.668916940689 -0.000112202587902 0.01\n",
      "92 300 -0.617753922939 -0.000112202587902 0.01\n",
      "92 350 -0.660245120525 -0.000112202587902 0.01\n",
      "92 400 -0.96350133419 -0.000112202587902 0.01\n",
      "92 450 -0.529548346996 -0.000112202587902 0.01\n",
      "92 500 -0.484787940979 -0.000112202587902 0.01\n",
      "92 550 -0.866153657436 -0.000112202587902 0.01\n",
      "valid_acc 82.10833333333333\n",
      "best valid_acc 82.10833333333333\n",
      "93 0 -0.384028822184 -0.000112202587902 0.01\n",
      "93 50 -0.695884525776 -0.000112202587902 0.01\n",
      "93 100 -0.625216424465 -0.000112202587902 0.01\n",
      "93 150 -0.514549553394 -0.000112202587902 0.01\n",
      "93 200 -0.452803701162 -0.000112202587902 0.01\n",
      "93 250 -0.595844209194 -0.000112202587902 0.01\n",
      "93 300 -0.813814938068 -0.000112202587902 0.01\n",
      "93 350 -1.18887019157 -0.000112202587902 0.01\n",
      "93 400 -0.844404935837 -0.000112202587902 0.01\n",
      "93 450 -0.638928890228 -0.000112202587902 0.01\n",
      "93 500 -0.656802356243 -0.000112202587902 0.01\n",
      "93 550 -0.554897785187 -0.000112202587902 0.01\n",
      "valid_acc 80.68333333333332\n",
      "94 0 -0.999397754669 -0.000112202587902 0.01\n",
      "94 50 -0.657495200634 -0.000112202587902 0.01\n",
      "94 100 -1.08697140217 -0.000112202587902 0.01\n",
      "94 150 -0.778695881367 -0.000112202587902 0.01\n",
      "94 200 -0.509756922722 -0.000112202587902 0.01\n",
      "94 250 -0.633532166481 -0.000112202587902 0.01\n",
      "94 300 -0.585760891438 -0.000112202587902 0.01\n",
      "94 350 -0.799131691456 -0.000112202587902 0.01\n",
      "94 400 -0.956277251244 -0.000112202587902 0.01\n",
      "94 450 -0.636597514153 -0.000112202587902 0.01\n",
      "94 500 -0.790436148643 -0.000112202587902 0.01\n",
      "94 550 -0.559734344482 -0.000112202587902 0.01\n",
      "valid_acc 80.66\n",
      "95 0 -0.441650956869 -0.000112202587902 0.01\n",
      "95 50 -0.472384184599 -0.000112202587902 0.01\n",
      "95 100 -0.707206726074 -0.000112202587902 0.01\n",
      "95 150 -0.539173543453 -0.000112202587902 0.01\n",
      "95 200 -0.359825730324 -0.000112202587902 0.01\n",
      "95 250 -0.531983971596 -0.000112202587902 0.01\n",
      "95 300 -0.415710031986 -0.000112202587902 0.01\n",
      "95 350 -0.896941840649 -0.000112202587902 0.01\n",
      "95 400 -0.553456962109 -0.000112202587902 0.01\n",
      "95 450 -0.746521472931 -0.000112202587902 0.01\n",
      "95 500 -0.364669948816 -0.000112202587902 0.01\n",
      "95 550 -0.710776090622 -0.000112202587902 0.01\n",
      "valid_acc 76.13499999999999\n",
      "96 0 -0.574728608131 -0.000112202587902 0.01\n",
      "96 50 -0.671935021877 -0.000112202587902 0.01\n",
      "96 100 -0.602599918842 -0.000112202587902 0.01\n",
      "96 150 -0.867576301098 -0.000112202587902 0.01\n",
      "96 200 -0.777688086033 -0.000112202587902 0.01\n",
      "96 250 -0.676168620586 -0.000112202587902 0.01\n",
      "96 300 -0.992850184441 -0.000112202587902 0.01\n",
      "96 350 -0.916916370392 -0.000112202587902 0.01\n",
      "96 400 -0.916846096516 -0.000112202587902 0.01\n",
      "96 450 -0.732795953751 -0.000112202587902 0.01\n",
      "96 500 -0.721053004265 -0.000112202587902 0.01\n",
      "96 550 -0.60587233305 -0.000112202587902 0.01\n",
      "valid_acc 79.94333333333333\n",
      "97 0 -0.367011725903 -0.000112202587902 0.01\n",
      "97 50 -0.781656503677 -0.000112202587902 0.01\n",
      "97 100 -0.719320058823 -0.000112202587902 0.01\n",
      "97 150 -0.901553034782 -0.000112202587902 0.01\n",
      "97 200 -0.316637486219 -0.000112202587902 0.01\n",
      "97 250 -0.59852796793 -0.000112202587902 0.01\n",
      "97 300 -0.845800042152 -0.000112202587902 0.01\n",
      "97 350 -0.710698306561 -0.000112202587902 0.01\n",
      "97 400 -0.558733463287 -0.000112202587902 0.01\n",
      "97 450 -0.589005351067 -0.000112202587902 0.01\n",
      "97 500 -0.662029206753 -0.000112202587902 0.01\n",
      "97 550 -0.708581984043 -0.000112202587902 0.01\n",
      "valid_acc 75.34833333333333\n",
      "98 0 -0.633505582809 -0.000112202587902 0.01\n",
      "98 50 -0.456730991602 -0.000112202587902 0.01\n",
      "98 100 -0.540552973747 -0.000112202587902 0.01\n",
      "98 150 -0.930781722069 -0.000112202587902 0.01\n",
      "98 200 -0.671853423119 -0.000112202587902 0.01\n",
      "98 250 -0.655983865261 -0.000112202587902 0.01\n",
      "98 300 -0.688347637653 -0.000112202587902 0.01\n",
      "98 350 -0.961328148842 -0.000112202587902 0.01\n",
      "98 400 -0.510823845863 -0.000112202587902 0.01\n",
      "98 450 -0.809262931347 -0.000112202587902 0.01\n",
      "98 500 -0.668790459633 -0.000112202587902 0.01\n",
      "98 550 -0.865238964558 -0.000112202587902 0.01\n",
      "valid_acc 77.00166666666667\n",
      "99 0 -0.75357657671 -0.000112202587902 0.01\n",
      "99 50 -0.672249078751 -0.000112202587902 0.01\n",
      "99 100 -0.684601068497 -0.000112202587902 0.01\n",
      "99 150 -0.535127520561 -0.000112202587902 0.01\n",
      "99 200 -0.61260253191 -0.000112202587902 0.01\n",
      "99 250 -0.804721057415 -0.000112202587902 0.01\n",
      "99 300 -1.10289132595 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 350 -0.398458987474 -0.000112202587902 0.01\n",
      "99 400 -1.54128706455 -0.000112202587902 0.01\n",
      "99 450 -0.624311625957 -0.000112202587902 0.01\n",
      "99 500 -0.935417115688 -0.000112202587902 0.01\n",
      "99 550 -0.72065538168 -0.000112202587902 0.01\n",
      "valid_acc 79.61666666666667\n",
      "100 0 -0.757225096226 -0.000112202587902 0.01\n",
      "100 50 -0.799174189568 -0.000112202587902 0.01\n",
      "100 100 -0.758721709251 -0.000112202587902 0.01\n",
      "100 150 -0.822801828384 -0.000112202587902 0.01\n",
      "100 200 -0.750170350075 -0.000112202587902 0.01\n",
      "100 250 -0.676130354404 -0.000112202587902 0.01\n",
      "100 300 -0.62860429287 -0.000112202587902 0.01\n",
      "100 350 -1.09087109566 -0.000112202587902 0.01\n",
      "100 400 -0.794941961765 -0.000112202587902 0.01\n",
      "100 450 -0.994907915592 -0.000112202587902 0.01\n",
      "100 500 -0.72499525547 -0.000112202587902 0.01\n",
      "100 550 -0.43608674407 -0.000112202587902 0.01\n",
      "valid_acc 79.46333333333332\n",
      "101 0 -0.922356426716 -0.000112202587902 0.01\n",
      "101 50 -0.952231585979 -0.000112202587902 0.01\n",
      "101 100 -0.911440134048 -0.000112202587902 0.01\n",
      "101 150 -1.61549496651 -0.000112202587902 0.01\n",
      "101 200 -0.488897353411 -0.000112202587902 0.01\n",
      "101 250 -0.616566956043 -0.000112202587902 0.01\n",
      "101 300 -0.906227588654 -0.000112202587902 0.01\n",
      "101 350 -0.617906928062 -0.000112202587902 0.01\n",
      "101 400 -0.643816173077 -0.000112202587902 0.01\n",
      "101 450 -0.890134751797 -0.000112202587902 0.01\n",
      "101 500 -0.76953458786 -0.000112202587902 0.01\n",
      "101 550 -0.78567481041 -0.000112202587902 0.01\n",
      "valid_acc 77.97166666666666\n",
      "102 0 -0.583911001682 -0.000112202587902 0.01\n",
      "102 50 -0.854093194008 -0.000112202587902 0.01\n",
      "102 100 -0.93592017889 -0.000112202587902 0.01\n",
      "102 150 -0.814607262611 -0.000112202587902 0.01\n",
      "102 200 -0.677044153214 -0.000112202587902 0.01\n",
      "102 250 -0.614930510521 -0.000112202587902 0.01\n",
      "102 300 -0.569574117661 -0.000112202587902 0.01\n",
      "102 350 -0.599118292332 -0.000112202587902 0.01\n",
      "102 400 -0.608964323997 -0.000112202587902 0.01\n",
      "102 450 -0.871760129929 -0.000112202587902 0.01\n",
      "102 500 -0.670793354511 -0.000112202587902 0.01\n",
      "102 550 -0.841699302197 -0.000112202587902 0.01\n",
      "valid_acc 75.88166666666667\n",
      "103 0 -0.648922204971 -0.000112202587902 0.01\n",
      "103 50 -0.486377179623 -0.000112202587902 0.01\n",
      "103 100 -0.836954653263 -0.000112202587902 0.01\n",
      "103 150 -0.663195669651 -0.000112202587902 0.01\n",
      "103 200 -0.793951630592 -0.000112202587902 0.01\n",
      "103 250 -0.654303371906 -0.000112202587902 0.01\n",
      "103 300 -0.73686671257 -0.000112202587902 0.01\n",
      "103 350 -0.601584017277 -0.000112202587902 0.01\n",
      "103 400 -0.580061674118 -0.000112202587902 0.01\n",
      "103 450 -0.62468111515 -0.000112202587902 0.01\n",
      "103 500 -0.787451803684 -0.000112202587902 0.01\n",
      "103 550 -0.613950133324 -0.000112202587902 0.01\n",
      "valid_acc 76.0\n",
      "104 0 -0.866639018059 -0.000112202587902 0.01\n",
      "104 50 -1.07875561714 -0.000112202587902 0.01\n",
      "104 100 -0.63956540823 -0.000112202587902 0.01\n",
      "104 150 -0.648612380028 -0.000112202587902 0.01\n",
      "104 200 -0.774795889854 -0.000112202587902 0.01\n",
      "104 250 -0.860827803612 -0.000112202587902 0.01\n",
      "104 300 -0.858194947243 -0.000112202587902 0.01\n",
      "104 350 -0.700610637665 -0.000112202587902 0.01\n",
      "104 400 -0.746627807617 -0.000112202587902 0.01\n",
      "104 450 -0.704252004623 -0.000112202587902 0.01\n",
      "104 500 -0.597062289715 -0.000112202587902 0.01\n",
      "104 550 -0.662548005581 -0.000112202587902 0.01\n",
      "valid_acc 78.15166666666667\n",
      "105 0 -0.618682801723 -0.000112202587902 0.01\n",
      "105 50 -0.751355588436 -0.000112202587902 0.01\n",
      "105 100 -0.841966152191 -0.000112202587902 0.01\n",
      "105 150 -0.72926312685 -0.000112202587902 0.01\n",
      "105 200 -0.969959497452 -0.000112202587902 0.01\n",
      "105 250 -0.441541224718 -0.000112202587902 0.01\n",
      "105 300 -0.696664750576 -0.000112202587902 0.01\n",
      "105 350 -0.650363326073 -0.000112202587902 0.01\n",
      "105 400 -0.749552309513 -0.000112202587902 0.01\n",
      "105 450 -0.851729869843 -0.000112202587902 0.01\n",
      "105 500 -0.701099574566 -0.000112202587902 0.01\n",
      "105 550 -0.643273234367 -0.000112202587902 0.01\n",
      "valid_acc 78.41833333333334\n",
      "106 0 -0.637039780617 -0.000112202587902 0.01\n",
      "106 50 -0.745099484921 -0.000112202587902 0.01\n",
      "106 100 -0.41595646739 -0.000112202587902 0.01\n",
      "106 150 -0.944743812084 -0.000112202587902 0.01\n",
      "106 200 -0.630763947964 -0.000112202587902 0.01\n",
      "106 250 -0.817989826202 -0.000112202587902 0.01\n",
      "106 300 -0.624151408672 -0.000112202587902 0.01\n",
      "106 350 -0.573096990585 -0.000112202587902 0.01\n",
      "106 400 -0.694537878036 -0.000112202587902 0.01\n",
      "106 450 -0.524537980556 -0.000112202587902 0.01\n",
      "106 500 -0.619910776615 -0.000112202587902 0.01\n",
      "106 550 -0.771689474583 -0.000112202587902 0.01\n",
      "valid_acc 78.5\n",
      "107 0 -0.787378370762 -0.000112202587902 0.01\n",
      "107 50 -0.455036342144 -0.000112202587902 0.01\n",
      "107 100 -0.830230355263 -0.000112202587902 0.01\n",
      "107 150 -0.927695631981 -0.000112202587902 0.01\n",
      "107 200 -0.871701478958 -0.000112202587902 0.01\n",
      "107 250 -0.856631875038 -0.000112202587902 0.01\n",
      "107 300 -0.756333291531 -0.000112202587902 0.01\n",
      "107 350 -0.97094553709 -0.000112202587902 0.01\n",
      "107 400 -0.590043067932 -0.000112202587902 0.01\n",
      "107 450 -0.673917710781 -0.000112202587902 0.01\n",
      "107 500 -0.748861789703 -0.000112202587902 0.01\n",
      "107 550 -0.493572622538 -0.000112202587902 0.01\n",
      "valid_acc 79.00500000000001\n",
      "108 0 -0.439297646284 -0.000112202587902 0.01\n",
      "108 50 -1.06411242485 -0.000112202587902 0.01\n",
      "108 100 -0.696004271507 -0.000112202587902 0.01\n",
      "108 150 -0.877692639828 -0.000112202587902 0.01\n",
      "108 200 -0.763458251953 -0.000112202587902 0.01\n",
      "108 250 -0.492880582809 -0.000112202587902 0.01\n",
      "108 300 -0.643258035183 -0.000112202587902 0.01\n",
      "108 350 -0.912714362144 -0.000112202587902 0.01\n",
      "108 400 -0.628151655197 -0.000112202587902 0.01\n",
      "108 450 -0.790017902851 -0.000112202587902 0.01\n",
      "108 500 -0.61243635416 -0.000112202587902 0.01\n",
      "108 550 -0.664186656475 -0.000112202587902 0.01\n",
      "valid_acc 77.69666666666667\n",
      "109 0 -0.700830459595 -0.000112202587902 0.01\n",
      "109 50 -0.947641372681 -0.000112202587902 0.01\n",
      "109 100 -0.77049934864 -0.000112202587902 0.01\n",
      "109 150 -0.512027800083 -0.000112202587902 0.01\n",
      "109 200 -0.497253954411 -0.000112202587902 0.01\n",
      "109 250 -0.489085733891 -0.000112202587902 0.01\n",
      "109 300 -0.691983163357 -0.000112202587902 0.01\n",
      "109 350 -0.859986662865 -0.000112202587902 0.01\n",
      "109 400 -0.878731548786 -0.000112202587902 0.01\n",
      "109 450 -0.698314726353 -0.000112202587902 0.01\n",
      "109 500 -0.553975999355 -0.000112202587902 0.01\n",
      "109 550 -0.964758455753 -0.000112202587902 0.01\n",
      "valid_acc 75.305\n",
      "110 0 -0.746419847012 -0.000112202587902 0.01\n",
      "110 50 -0.626801013947 -0.000112202587902 0.01\n",
      "110 100 -0.695250093937 -0.000112202587902 0.01\n",
      "110 150 -0.701047837734 -0.000112202587902 0.01\n",
      "110 200 -0.45952796936 -0.000112202587902 0.01\n",
      "110 250 -0.943508148193 -0.000112202587902 0.01\n",
      "110 300 -0.592183470726 -0.000112202587902 0.01\n",
      "110 350 -0.517193078995 -0.000112202587902 0.01\n",
      "110 400 -0.603249132633 -0.000112202587902 0.01\n",
      "110 450 -0.716463148594 -0.000112202587902 0.01\n",
      "110 500 -0.556376934052 -0.000112202587902 0.01\n",
      "110 550 -0.701808750629 -0.000112202587902 0.01\n",
      "valid_acc 78.53\n",
      "111 0 -0.592175245285 -0.000112202587902 0.01\n",
      "111 50 -0.696191191673 -0.000112202587902 0.01\n",
      "111 100 -0.70100992918 -0.000112202587902 0.01\n",
      "111 150 -0.797868132591 -0.000112202587902 0.01\n",
      "111 200 -1.08493828773 -0.000112202587902 0.01\n",
      "111 250 -0.774304509163 -0.000112202587902 0.01\n",
      "111 300 -0.617121875286 -0.000112202587902 0.01\n",
      "111 350 -0.974508583546 -0.000112202587902 0.01\n",
      "111 400 -1.01898884773 -0.000112202587902 0.01\n",
      "111 450 -1.41822886467 -0.000112202587902 0.01\n",
      "111 500 -0.608646512032 -0.000112202587902 0.01\n",
      "111 550 -0.820364713669 -0.000112202587902 0.01\n",
      "valid_acc 76.55999999999999\n",
      "112 0 -0.534847199917 -0.000112202587902 0.01\n",
      "112 50 -0.836295366287 -0.000112202587902 0.01\n",
      "112 100 -1.12258541584 -0.000112202587902 0.01\n",
      "112 150 -0.80717498064 -0.000112202587902 0.01\n",
      "112 200 -0.815969288349 -0.000112202587902 0.01\n",
      "112 250 -0.65654528141 -0.000112202587902 0.01\n",
      "112 300 -0.698454618454 -0.000112202587902 0.01\n",
      "112 350 -0.91737395525 -0.000112202587902 0.01\n",
      "112 400 -0.585757017136 -0.000112202587902 0.01\n",
      "112 450 -0.675713598728 -0.000112202587902 0.01\n",
      "112 500 -0.64382982254 -0.000112202587902 0.01\n",
      "112 550 -0.815305113792 -0.000112202587902 0.01\n",
      "valid_acc 77.41166666666666\n",
      "113 0 -0.756367206573 -0.000112202587902 0.01\n",
      "113 50 -0.733718395233 -0.000112202587902 0.01\n",
      "113 100 -1.13609850407 -0.000112202587902 0.01\n",
      "113 150 -0.765248417854 -0.000112202587902 0.01\n",
      "113 200 -1.01108825207 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113 250 -0.675662815571 -0.000112202587902 0.01\n",
      "113 300 -0.461668998003 -0.000112202587902 0.01\n",
      "113 350 -0.470628321171 -0.000112202587902 0.01\n",
      "113 400 -0.561836004257 -0.000112202587902 0.01\n",
      "113 450 -0.796682596207 -0.000112202587902 0.01\n",
      "113 500 -0.581245064735 -0.000112202587902 0.01\n",
      "113 550 -0.717346429825 -0.000112202587902 0.01\n",
      "valid_acc 76.5\n",
      "114 0 -0.671857833862 -0.000112202587902 0.01\n",
      "114 50 -0.779184937477 -0.000112202587902 0.01\n",
      "114 100 -0.652430713177 -0.000112202587902 0.01\n",
      "114 150 -1.05456638336 -0.000112202587902 0.01\n",
      "114 200 -0.546969234943 -0.000112202587902 0.01\n",
      "114 250 -0.772476673126 -0.000112202587902 0.01\n",
      "114 300 -0.865570008755 -0.000112202587902 0.01\n",
      "114 350 -0.502821028233 -0.000112202587902 0.01\n",
      "114 400 -0.733815252781 -0.000112202587902 0.01\n",
      "114 450 -0.73941886425 -0.000112202587902 0.01\n",
      "114 500 -0.876294851303 -0.000112202587902 0.01\n",
      "114 550 -0.711953103542 -0.000112202587902 0.01\n",
      "valid_acc 78.89\n",
      "115 0 -0.776031434536 -0.000112202587902 0.01\n",
      "115 50 -0.732080161572 -0.000112202587902 0.01\n",
      "115 100 -0.6074847579 -0.000112202587902 0.01\n",
      "115 150 -0.962619066238 -0.000112202587902 0.01\n",
      "115 200 -0.710499405861 -0.000112202587902 0.01\n",
      "115 250 -0.641085684299 -0.000112202587902 0.01\n",
      "115 300 -0.704977810383 -0.000112202587902 0.01\n",
      "115 350 -0.667196333408 -0.000112202587902 0.01\n",
      "115 400 -0.671921014786 -0.000112202587902 0.01\n",
      "115 450 -0.847623348236 -0.000112202587902 0.01\n",
      "115 500 -1.00266361237 -0.000112202587902 0.01\n",
      "115 550 -0.781105577946 -0.000112202587902 0.01\n",
      "valid_acc 78.7\n",
      "116 0 -0.940093815327 -0.000112202587902 0.01\n",
      "116 50 -0.785518348217 -0.000112202587902 0.01\n",
      "116 100 -0.7617841959 -0.000112202587902 0.01\n",
      "116 150 -0.598882436752 -0.000112202587902 0.01\n",
      "116 200 -0.891270577908 -0.000112202587902 0.01\n",
      "116 250 -0.604455411434 -0.000112202587902 0.01\n",
      "116 300 -0.68378251791 -0.000112202587902 0.01\n",
      "116 350 -0.47506248951 -0.000112202587902 0.01\n",
      "116 400 -0.828326642513 -0.000112202587902 0.01\n",
      "116 450 -0.62879961729 -0.000112202587902 0.01\n",
      "116 500 -0.605786025524 -0.000112202587902 0.01\n",
      "116 550 -1.25189495087 -0.000112202587902 0.01\n",
      "valid_acc 78.23166666666667\n",
      "117 0 -0.850760817528 -0.000112202587902 0.01\n",
      "117 50 -0.814226865768 -0.000112202587902 0.01\n",
      "117 100 -0.9230889678 -0.000112202587902 0.01\n",
      "117 150 -0.692719101906 -0.000112202587902 0.01\n",
      "117 200 -0.662277281284 -0.000112202587902 0.01\n",
      "117 250 -1.01368927956 -0.000112202587902 0.01\n",
      "117 300 -0.695820868015 -0.000112202587902 0.01\n",
      "117 350 -0.861282408237 -0.000112202587902 0.01\n",
      "117 400 -0.501082479954 -0.000112202587902 0.01\n",
      "117 450 -1.20761394501 -0.000112202587902 0.01\n",
      "117 500 -0.635275781155 -0.000112202587902 0.01\n",
      "117 550 -0.73887604475 -0.000112202587902 0.01\n",
      "valid_acc 74.45333333333333\n",
      "118 0 -0.824274837971 -0.000112202587902 0.01\n",
      "118 50 -0.756035387516 -0.000112202587902 0.01\n",
      "118 100 -0.478772968054 -0.000112202587902 0.01\n",
      "118 150 -0.815357208252 -0.000112202587902 0.01\n",
      "118 200 -0.874589383602 -0.000112202587902 0.01\n",
      "118 250 -0.656307160854 -0.000112202587902 0.01\n",
      "118 300 -0.545653700829 -0.000112202587902 0.01\n",
      "118 350 -0.604452192783 -0.000112202587902 0.01\n",
      "118 400 -0.519868135452 -0.000112202587902 0.01\n",
      "118 450 -0.547393798828 -0.000112202587902 0.01\n",
      "118 500 -0.759097874165 -0.000112202587902 0.01\n",
      "118 550 -0.885517656803 -0.000112202587902 0.01\n",
      "valid_acc 76.83833333333332\n",
      "119 0 -0.493021607399 -0.000112202587902 0.01\n",
      "119 50 -0.842741787434 -0.000112202587902 0.01\n",
      "119 100 -0.794794857502 -0.000112202587902 0.01\n",
      "119 150 -0.881246089935 -0.000112202587902 0.01\n",
      "119 200 -0.80867433548 -0.000112202587902 0.01\n",
      "119 250 -1.07819736004 -0.000112202587902 0.01\n",
      "119 300 -0.743597328663 -0.000112202587902 0.01\n",
      "119 350 -1.05925405025 -0.000112202587902 0.01\n",
      "119 400 -0.839674532413 -0.000112202587902 0.01\n",
      "119 450 -0.549799323082 -0.000112202587902 0.01\n",
      "119 500 -0.859770953655 -0.000112202587902 0.01\n",
      "119 550 -0.747847735882 -0.000112202587902 0.01\n",
      "valid_acc 80.43333333333334\n",
      "120 0 -1.05804240704 -0.000112202587902 0.01\n",
      "120 50 -0.559625685215 -0.000112202587902 0.01\n",
      "120 100 -0.608637332916 -0.000112202587902 0.01\n",
      "120 150 -0.975741505623 -0.000112202587902 0.01\n",
      "120 200 -0.785477876663 -0.000112202587902 0.01\n",
      "120 250 -0.49186232686 -0.000112202587902 0.01\n",
      "120 300 -0.680782675743 -0.000112202587902 0.01\n",
      "120 350 -0.518432676792 -0.000112202587902 0.01\n",
      "120 400 -0.689981639385 -0.000112202587902 0.01\n",
      "120 450 -0.700106143951 -0.000112202587902 0.01\n",
      "120 500 -0.649434745312 -0.000112202587902 0.01\n",
      "120 550 -0.620126724243 -0.000112202587902 0.01\n",
      "valid_acc 77.06833333333334\n",
      "121 0 -0.792594671249 -0.000112202587902 0.01\n",
      "121 50 -1.02191007137 -0.000112202587902 0.01\n",
      "121 100 -0.984181344509 -0.000112202587902 0.01\n",
      "121 150 -0.613662362099 -0.000112202587902 0.01\n",
      "121 200 -0.718294680119 -0.000112202587902 0.01\n",
      "121 250 -0.600390911102 -0.000112202587902 0.01\n",
      "121 300 -0.532881259918 -0.000112202587902 0.01\n",
      "121 350 -0.718459904194 -0.000112202587902 0.01\n",
      "121 400 -0.925384521484 -0.000112202587902 0.01\n",
      "121 450 -0.934175729752 -0.000112202587902 0.01\n",
      "121 500 -0.937138497829 -0.000112202587902 0.01\n",
      "121 550 -0.829164862633 -0.000112202587902 0.01\n",
      "valid_acc 76.96166666666666\n",
      "122 0 -0.645421087742 -0.000112202587902 0.01\n",
      "122 50 -0.761998534203 -0.000112202587902 0.01\n",
      "122 100 -0.435172915459 -0.000112202587902 0.01\n",
      "122 150 -0.754373371601 -0.000112202587902 0.01\n",
      "122 200 -0.611208856106 -0.000112202587902 0.01\n",
      "122 250 -0.775643765926 -0.000112202587902 0.01\n",
      "122 300 -0.514427363873 -0.000112202587902 0.01\n",
      "122 350 -0.943327009678 -0.000112202587902 0.01\n",
      "122 400 -0.897017419338 -0.000112202587902 0.01\n",
      "122 450 -0.880246400833 -0.000112202587902 0.01\n",
      "122 500 -0.79277831316 -0.000112202587902 0.01\n",
      "122 550 -1.03636658192 -0.000112202587902 0.01\n",
      "valid_acc 79.16499999999999\n",
      "123 0 -1.19541358948 -0.000112202587902 0.01\n",
      "123 50 -0.865168333054 -0.000112202587902 0.01\n",
      "123 100 -0.525443971157 -0.000112202587902 0.01\n",
      "123 150 -0.677754819393 -0.000112202587902 0.01\n",
      "123 200 -0.491393953562 -0.000112202587902 0.01\n",
      "123 250 -0.959556341171 -0.000112202587902 0.01\n",
      "123 300 -0.840098202229 -0.000112202587902 0.01\n",
      "123 350 -0.794399738312 -0.000112202587902 0.01\n",
      "123 400 -0.452122956514 -0.000112202587902 0.01\n",
      "123 450 -0.442245125771 -0.000112202587902 0.01\n",
      "123 500 -0.529766738415 -0.000112202587902 0.01\n",
      "123 550 -0.993807435036 -0.000112202587902 0.01\n",
      "valid_acc 79.65833333333333\n",
      "124 0 -0.580281376839 -0.000112202587902 0.01\n",
      "124 50 -0.825440526009 -0.000112202587902 0.01\n",
      "124 100 -0.53210401535 -0.000112202587902 0.01\n",
      "124 150 -0.696139812469 -0.000112202587902 0.01\n",
      "124 200 -0.648417592049 -0.000112202587902 0.01\n",
      "124 250 -0.568937897682 -0.000112202587902 0.01\n",
      "124 300 -0.535877943039 -0.000112202587902 0.01\n",
      "124 350 -0.753121256828 -0.000112202587902 0.01\n",
      "124 400 -0.536480665207 -0.000112202587902 0.01\n",
      "124 450 -0.906271338463 -0.000112202587902 0.01\n",
      "124 500 -0.946200728416 -0.000112202587902 0.01\n",
      "124 550 -0.859651327133 -0.000112202587902 0.01\n",
      "valid_acc 78.565\n",
      "125 0 -0.774693965912 -0.000112202587902 0.01\n",
      "125 50 -0.822368443012 -0.000112202587902 0.01\n",
      "125 100 -0.818372547626 -0.000112202587902 0.01\n",
      "125 150 -0.853851675987 -0.000112202587902 0.01\n",
      "125 200 -0.663974583149 -0.000112202587902 0.01\n",
      "125 250 -0.815656840801 -0.000112202587902 0.01\n",
      "125 300 -0.628115832806 -0.000112202587902 0.01\n",
      "125 350 -0.646585464478 -0.000112202587902 0.01\n",
      "125 400 -0.343869924545 -0.000112202587902 0.01\n",
      "125 450 -0.490205675364 -0.000112202587902 0.01\n",
      "125 500 -0.775145292282 -0.000112202587902 0.01\n",
      "125 550 -0.712315797806 -0.000112202587902 0.01\n",
      "valid_acc 77.13833333333334\n",
      "126 0 -0.621256530285 -0.000112202587902 0.01\n",
      "126 50 -0.602845966816 -0.000112202587902 0.01\n",
      "126 100 -0.61428129673 -0.000112202587902 0.01\n",
      "126 150 -0.722075581551 -0.000112202587902 0.01\n",
      "126 200 -0.514795899391 -0.000112202587902 0.01\n",
      "126 250 -0.660625755787 -0.000112202587902 0.01\n",
      "126 300 -1.05134344101 -0.000112202587902 0.01\n",
      "126 350 -0.619568586349 -0.000112202587902 0.01\n",
      "126 400 -0.560233354568 -0.000112202587902 0.01\n",
      "126 450 -0.508626818657 -0.000112202587902 0.01\n",
      "126 500 -0.708773553371 -0.000112202587902 0.01\n",
      "126 550 -0.500457525253 -0.000112202587902 0.01\n",
      "valid_acc 78.77\n",
      "127 0 -0.602201402187 -0.000112202587902 0.01\n",
      "127 50 -0.768311321735 -0.000112202587902 0.01\n",
      "127 100 -0.489119797945 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127 150 -0.724050998688 -0.000112202587902 0.01\n",
      "127 200 -0.625877857208 -0.000112202587902 0.01\n",
      "127 250 -0.805654346943 -0.000112202587902 0.01\n",
      "127 300 -0.505161523819 -0.000112202587902 0.01\n",
      "127 350 -0.646341621876 -0.000112202587902 0.01\n",
      "127 400 -0.83058565855 -0.000112202587902 0.01\n",
      "127 450 -0.799794316292 -0.000112202587902 0.01\n",
      "127 500 -0.675758719444 -0.000112202587902 0.01\n",
      "127 550 -0.705966055393 -0.000112202587902 0.01\n",
      "valid_acc 79.11166666666666\n",
      "128 0 -0.535241544247 -0.000112202587902 0.01\n",
      "128 50 -0.823708355427 -0.000112202587902 0.01\n",
      "128 100 -0.906671762466 -0.000112202587902 0.01\n",
      "128 150 -0.580531418324 -0.000112202587902 0.01\n",
      "128 200 -0.737941384315 -0.000112202587902 0.01\n",
      "128 250 -0.502165555954 -0.000112202587902 0.01\n",
      "128 300 -1.09701538086 -0.000112202587902 0.01\n",
      "128 350 -0.574063658714 -0.000112202587902 0.01\n",
      "128 400 -0.903064668179 -0.000112202587902 0.01\n",
      "128 450 -0.644606769085 -0.000112202587902 0.01\n",
      "128 500 -0.415323257446 -0.000112202587902 0.01\n",
      "128 550 -0.760811388493 -0.000112202587902 0.01\n",
      "valid_acc 78.81\n",
      "129 0 -0.599320590496 -0.000112202587902 0.01\n",
      "129 50 -0.651896417141 -0.000112202587902 0.01\n",
      "129 100 -0.594145476818 -0.000112202587902 0.01\n",
      "129 150 -0.667169332504 -0.000112202587902 0.01\n",
      "129 200 -1.03094911575 -0.000112202587902 0.01\n",
      "129 250 -0.627045989037 -0.000112202587902 0.01\n",
      "129 300 -0.983385324478 -0.000112202587902 0.01\n",
      "129 350 -0.528672039509 -0.000112202587902 0.01\n",
      "129 400 -0.485918134451 -0.000112202587902 0.01\n",
      "129 450 -0.855561196804 -0.000112202587902 0.01\n",
      "129 500 -0.887575209141 -0.000112202587902 0.01\n",
      "129 550 -0.726155042648 -0.000112202587902 0.01\n",
      "valid_acc 78.51666666666667\n",
      "130 0 -0.667459845543 -0.000112202587902 0.01\n",
      "130 50 -1.06869363785 -0.000112202587902 0.01\n",
      "130 100 -0.552662849426 -0.000112202587902 0.01\n",
      "130 150 -0.821685492992 -0.000112202587902 0.01\n",
      "130 200 -0.699150264263 -0.000112202587902 0.01\n",
      "130 250 -0.684483468533 -0.000112202587902 0.01\n",
      "130 300 -0.720298409462 -0.000112202587902 0.01\n",
      "130 350 -0.919368386269 -0.000112202587902 0.01\n",
      "130 400 -0.891285121441 -0.000112202587902 0.01\n",
      "130 450 -0.585625052452 -0.000112202587902 0.01\n",
      "130 500 -0.471250385046 -0.000112202587902 0.01\n",
      "130 550 -0.671727836132 -0.000112202587902 0.01\n",
      "valid_acc 79.99666666666667\n",
      "131 0 -0.422627151012 -0.000112202587902 0.01\n",
      "131 50 -0.754972398281 -0.000112202587902 0.01\n",
      "131 100 -0.790392994881 -0.000112202587902 0.01\n",
      "131 150 -0.839286506176 -0.000112202587902 0.01\n",
      "131 200 -0.601431369781 -0.000112202587902 0.01\n",
      "131 250 -0.506727337837 -0.000112202587902 0.01\n",
      "131 300 -0.787336170673 -0.000112202587902 0.01\n",
      "131 350 -0.89851051569 -0.000112202587902 0.01\n",
      "131 400 -0.924588561058 -0.000112202587902 0.01\n",
      "131 450 -0.551209688187 -0.000112202587902 0.01\n",
      "131 500 -0.827255189419 -0.000112202587902 0.01\n",
      "131 550 -0.638941884041 -0.000112202587902 0.01\n",
      "valid_acc 75.49833333333333\n",
      "132 0 -0.737945020199 -0.000112202587902 0.01\n",
      "132 50 -0.902930676937 -0.000112202587902 0.01\n",
      "132 100 -0.636667847633 -0.000112202587902 0.01\n",
      "132 150 -0.75530731678 -0.000112202587902 0.01\n",
      "132 200 -0.859326004982 -0.000112202587902 0.01\n",
      "132 250 -0.843156337738 -0.000112202587902 0.01\n",
      "132 300 -0.654948115349 -0.000112202587902 0.01\n",
      "132 350 -0.84444963932 -0.000112202587902 0.01\n",
      "132 400 -0.787731170654 -0.000112202587902 0.01\n",
      "132 450 -0.388704836369 -0.000112202587902 0.01\n",
      "132 500 -0.849605083466 -0.000112202587902 0.01\n",
      "132 550 -1.03829407692 -0.000112202587902 0.01\n",
      "valid_acc 77.91666666666667\n",
      "133 0 -0.883039176464 -0.000112202587902 0.01\n",
      "133 50 -0.844391047955 -0.000112202587902 0.01\n",
      "133 100 -0.938567399979 -0.000112202587902 0.01\n",
      "133 150 -0.70800191164 -0.000112202587902 0.01\n",
      "133 200 -0.642635822296 -0.000112202587902 0.01\n",
      "133 250 -0.492491722107 -0.000112202587902 0.01\n",
      "133 300 -0.579993486404 -0.000112202587902 0.01\n",
      "133 350 -0.588456690311 -0.000112202587902 0.01\n",
      "133 400 -0.661479234695 -0.000112202587902 0.01\n",
      "133 450 -0.411075681448 -0.000112202587902 0.01\n",
      "133 500 -0.639968276024 -0.000112202587902 0.01\n",
      "133 550 -0.48821875453 -0.000112202587902 0.01\n",
      "valid_acc 76.29333333333334\n",
      "134 0 -0.54008680582 -0.000112202587902 0.01\n",
      "134 50 -0.6554453969 -0.000112202587902 0.01\n",
      "134 100 -0.54179251194 -0.000112202587902 0.01\n",
      "134 150 -0.729065239429 -0.000112202587902 0.01\n",
      "134 200 -0.569815814495 -0.000112202587902 0.01\n",
      "134 250 -0.845355272293 -0.000112202587902 0.01\n",
      "134 300 -0.472777456045 -0.000112202587902 0.01\n",
      "134 350 -0.602242588997 -0.000112202587902 0.01\n",
      "134 400 -0.508808016777 -0.000112202587902 0.01\n",
      "134 450 -0.855793774128 -0.000112202587902 0.01\n",
      "134 500 -0.688978791237 -0.000112202587902 0.01\n",
      "134 550 -0.599760711193 -0.000112202587902 0.01\n",
      "valid_acc 76.95166666666667\n",
      "135 0 -0.540538728237 -0.000112202587902 0.01\n",
      "135 50 -0.826174259186 -0.000112202587902 0.01\n",
      "135 100 -0.647478044033 -0.000112202587902 0.01\n",
      "135 150 -0.911099016666 -0.000112202587902 0.01\n",
      "135 200 -0.57682043314 -0.000112202587902 0.01\n",
      "135 250 -0.700525343418 -0.000112202587902 0.01\n",
      "135 300 -0.671746373177 -0.000112202587902 0.01\n",
      "135 350 -0.956077516079 -0.000112202587902 0.01\n",
      "135 400 -0.857900440693 -0.000112202587902 0.01\n",
      "135 450 -0.789672255516 -0.000112202587902 0.01\n",
      "135 500 -0.620608091354 -0.000112202587902 0.01\n",
      "135 550 -0.444805920124 -0.000112202587902 0.01\n",
      "valid_acc 76.84833333333333\n",
      "136 0 -0.650726914406 -0.000112202587902 0.01\n",
      "136 50 -0.420950472355 -0.000112202587902 0.01\n",
      "136 100 -0.727341294289 -0.000112202587902 0.01\n",
      "136 150 -0.767651379108 -0.000112202587902 0.01\n",
      "136 200 -0.650767207146 -0.000112202587902 0.01\n",
      "136 250 -0.773166596889 -0.000112202587902 0.01\n",
      "136 300 -0.689282774925 -0.000112202587902 0.01\n",
      "136 350 -0.763277232647 -0.000112202587902 0.01\n",
      "136 400 -0.708701610565 -0.000112202587902 0.01\n",
      "136 450 -0.776887834072 -0.000112202587902 0.01\n",
      "136 500 -0.476750969887 -0.000112202587902 0.01\n",
      "136 550 -0.669529139996 -0.000112202587902 0.01\n",
      "valid_acc 79.21166666666667\n",
      "137 0 -0.392747312784 -0.000112202587902 0.01\n",
      "137 50 -0.511272609234 -0.000112202587902 0.01\n",
      "137 100 -0.775567770004 -0.000112202587902 0.01\n",
      "137 150 -0.556106209755 -0.000112202587902 0.01\n",
      "137 200 -0.642069756985 -0.000112202587902 0.01\n",
      "137 250 -0.724645674229 -0.000112202587902 0.01\n",
      "137 300 -0.818530023098 -0.000112202587902 0.01\n",
      "137 350 -0.51966047287 -0.000112202587902 0.01\n",
      "137 400 -0.803609848022 -0.000112202587902 0.01\n",
      "137 450 -0.892015516758 -0.000112202587902 0.01\n",
      "137 500 -0.921561658382 -0.000112202587902 0.01\n",
      "137 550 -0.715708613396 -0.000112202587902 0.01\n",
      "valid_acc 78.80166666666668\n",
      "138 0 -0.680048286915 -0.000112202587902 0.01\n",
      "138 50 -1.03787791729 -0.000112202587902 0.01\n",
      "138 100 -0.378321230412 -0.000112202587902 0.01\n",
      "138 150 -0.521626532078 -0.000112202587902 0.01\n",
      "138 200 -0.954167962074 -0.000112202587902 0.01\n",
      "138 250 -0.514553844929 -0.000112202587902 0.01\n",
      "138 300 -0.745021224022 -0.000112202587902 0.01\n",
      "138 350 -0.803774356842 -0.000112202587902 0.01\n",
      "138 400 -0.914835810661 -0.000112202587902 0.01\n",
      "138 450 -0.906614124775 -0.000112202587902 0.01\n",
      "138 500 -0.639351248741 -0.000112202587902 0.01\n",
      "138 550 -0.855569779873 -0.000112202587902 0.01\n",
      "valid_acc 81.17166666666667\n",
      "139 0 -0.459988325834 -0.000112202587902 0.01\n",
      "139 50 -0.380957901478 -0.000112202587902 0.01\n",
      "139 100 -0.47775632143 -0.000112202587902 0.01\n",
      "139 150 -0.708090662956 -0.000112202587902 0.01\n",
      "139 200 -0.738510131836 -0.000112202587902 0.01\n",
      "139 250 -0.708383500576 -0.000112202587902 0.01\n",
      "139 300 -0.821145534515 -0.000112202587902 0.01\n",
      "139 350 -0.770249724388 -0.000112202587902 0.01\n",
      "139 400 -0.687927305698 -0.000112202587902 0.01\n",
      "139 450 -0.651322841644 -0.000112202587902 0.01\n",
      "139 500 -0.882727861404 -0.000112202587902 0.01\n",
      "139 550 -0.655575037003 -0.000112202587902 0.01\n",
      "valid_acc 76.58166666666666\n",
      "140 0 -0.550624668598 -0.000112202587902 0.01\n",
      "140 50 -0.771954119205 -0.000112202587902 0.01\n",
      "140 100 -0.8407304883 -0.000112202587902 0.01\n",
      "140 150 -0.575731396675 -0.000112202587902 0.01\n",
      "140 200 -0.559349954128 -0.000112202587902 0.01\n",
      "140 250 -0.497379988432 -0.000112202587902 0.01\n",
      "140 300 -0.547770261765 -0.000112202587902 0.01\n",
      "140 350 -0.714731395245 -0.000112202587902 0.01\n",
      "140 400 -0.642054498196 -0.000112202587902 0.01\n",
      "140 450 -1.09585678577 -0.000112202587902 0.01\n",
      "140 500 -0.867447733879 -0.000112202587902 0.01\n",
      "140 550 -0.630410671234 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_acc 73.595\n",
      "141 0 -0.558148741722 -0.000112202587902 0.01\n",
      "141 50 -0.64916074276 -0.000112202587902 0.01\n",
      "141 100 -0.637393176556 -0.000112202587902 0.01\n",
      "141 150 -0.523883879185 -0.000112202587902 0.01\n",
      "141 200 -0.801377415657 -0.000112202587902 0.01\n",
      "141 250 -0.574195027351 -0.000112202587902 0.01\n",
      "141 300 -0.612527370453 -0.000112202587902 0.01\n",
      "141 350 -0.804567575455 -0.000112202587902 0.01\n",
      "141 400 -0.468449473381 -0.000112202587902 0.01\n",
      "141 450 -0.778932332993 -0.000112202587902 0.01\n",
      "141 500 -0.830764591694 -0.000112202587902 0.01\n",
      "141 550 -0.664522767067 -0.000112202587902 0.01\n",
      "valid_acc 79.475\n",
      "142 0 -0.640269339085 -0.000112202587902 0.01\n",
      "142 50 -0.983405351639 -0.000112202587902 0.01\n",
      "142 100 -0.582395493984 -0.000112202587902 0.01\n",
      "142 150 -0.583970487118 -0.000112202587902 0.01\n",
      "142 200 -0.522435247898 -0.000112202587902 0.01\n",
      "142 250 -0.382485687733 -0.000112202587902 0.01\n",
      "142 300 -0.50391793251 -0.000112202587902 0.01\n",
      "142 350 -0.439417809248 -0.000112202587902 0.01\n",
      "142 400 -0.529361963272 -0.000112202587902 0.01\n",
      "142 450 -0.521515607834 -0.000112202587902 0.01\n",
      "142 500 -0.625547587872 -0.000112202587902 0.01\n",
      "142 550 -0.534828782082 -0.000112202587902 0.01\n",
      "valid_acc 78.27\n",
      "143 0 -0.598619222641 -0.000112202587902 0.01\n",
      "143 50 -0.509462416172 -0.000112202587902 0.01\n",
      "143 100 -0.534568548203 -0.000112202587902 0.01\n",
      "143 150 -0.517339527607 -0.000112202587902 0.01\n",
      "143 200 -0.761258721352 -0.000112202587902 0.01\n",
      "143 250 -0.79544532299 -0.000112202587902 0.01\n",
      "143 300 -0.593296349049 -0.000112202587902 0.01\n",
      "143 350 -0.443902313709 -0.000112202587902 0.01\n",
      "143 400 -0.6968780756 -0.000112202587902 0.01\n",
      "143 450 -0.581116080284 -0.000112202587902 0.01\n",
      "143 500 -0.860813081264 -0.000112202587902 0.01\n",
      "143 550 -0.469981312752 -0.000112202587902 0.01\n",
      "valid_acc 75.42\n",
      "144 0 -1.0587502718 -0.000112202587902 0.01\n",
      "144 50 -0.899404466152 -0.000112202587902 0.01\n",
      "144 100 -0.633127093315 -0.000112202587902 0.01\n",
      "144 150 -0.577622175217 -0.000112202587902 0.01\n",
      "144 200 -0.731029808521 -0.000112202587902 0.01\n",
      "144 250 -0.612479448318 -0.000112202587902 0.01\n",
      "144 300 -0.858995676041 -0.000112202587902 0.01\n",
      "144 350 -0.712817966938 -0.000112202587902 0.01\n",
      "144 400 -0.957461714745 -0.000112202587902 0.01\n",
      "144 450 -0.60857385397 -0.000112202587902 0.01\n",
      "144 500 -0.682745277882 -0.000112202587902 0.01\n",
      "144 550 -0.760456860065 -0.000112202587902 0.01\n",
      "valid_acc 78.41166666666666\n",
      "145 0 -0.677281439304 -0.000112202587902 0.01\n",
      "145 50 -0.717970728874 -0.000112202587902 0.01\n",
      "145 100 -0.722935736179 -0.000112202587902 0.01\n",
      "145 150 -0.690325796604 -0.000112202587902 0.01\n",
      "145 200 -0.679768979549 -0.000112202587902 0.01\n",
      "145 250 -0.894617378712 -0.000112202587902 0.01\n",
      "145 300 -0.805370926857 -0.000112202587902 0.01\n",
      "145 350 -0.821033835411 -0.000112202587902 0.01\n",
      "145 400 -0.612805306911 -0.000112202587902 0.01\n",
      "145 450 -0.530359625816 -0.000112202587902 0.01\n",
      "145 500 -0.54159039259 -0.000112202587902 0.01\n",
      "145 550 -0.462392508984 -0.000112202587902 0.01\n",
      "valid_acc 76.70666666666666\n",
      "146 0 -1.01590037346 -0.000112202587902 0.01\n",
      "146 50 -0.603930830956 -0.000112202587902 0.01\n",
      "146 100 -0.716363549232 -0.000112202587902 0.01\n",
      "146 150 -0.819755554199 -0.000112202587902 0.01\n",
      "146 200 -0.799127340317 -0.000112202587902 0.01\n",
      "146 250 -0.631971001625 -0.000112202587902 0.01\n",
      "146 300 -0.960988998413 -0.000112202587902 0.01\n",
      "146 350 -0.677005589008 -0.000112202587902 0.01\n",
      "146 400 -0.740251839161 -0.000112202587902 0.01\n",
      "146 450 -1.27678954601 -0.000112202587902 0.01\n",
      "146 500 -0.444201499224 -0.000112202587902 0.01\n",
      "146 550 -0.84935849905 -0.000112202587902 0.01\n",
      "valid_acc 69.55833333333334\n",
      "147 0 -0.411408245564 -0.000112202587902 0.01\n",
      "147 50 -0.980782449245 -0.000112202587902 0.01\n",
      "147 100 -0.528055608273 -0.000112202587902 0.01\n",
      "147 150 -0.760694980621 -0.000112202587902 0.01\n",
      "147 200 -0.58501124382 -0.000112202587902 0.01\n",
      "147 250 -0.924777626991 -0.000112202587902 0.01\n",
      "147 300 -0.865651786327 -0.000112202587902 0.01\n",
      "147 350 -0.559518098831 -0.000112202587902 0.01\n",
      "147 400 -0.66464060545 -0.000112202587902 0.01\n",
      "147 450 -0.8003937006 -0.000112202587902 0.01\n",
      "147 500 -0.627775549889 -0.000112202587902 0.01\n",
      "147 550 -0.731505274773 -0.000112202587902 0.01\n",
      "valid_acc 78.46499999999999\n",
      "148 0 -0.778849482536 -0.000112202587902 0.01\n",
      "148 50 -0.334699213505 -0.000112202587902 0.01\n",
      "148 100 -0.534599542618 -0.000112202587902 0.01\n",
      "148 150 -0.563726305962 -0.000112202587902 0.01\n",
      "148 200 -0.592477619648 -0.000112202587902 0.01\n",
      "148 250 -0.636410951614 -0.000112202587902 0.01\n",
      "148 300 -0.526425540447 -0.000112202587902 0.01\n",
      "148 350 -0.655018985271 -0.000112202587902 0.01\n",
      "148 400 -1.36114108562 -0.000112202587902 0.01\n",
      "148 450 -0.815556704998 -0.000112202587902 0.01\n",
      "148 500 -0.882742404938 -0.000112202587902 0.01\n",
      "148 550 -0.994147717953 -0.000112202587902 0.01\n",
      "valid_acc 78.30666666666667\n",
      "149 0 -0.591682314873 -0.000112202587902 0.01\n",
      "149 50 -0.523750245571 -0.000112202587902 0.01\n",
      "149 100 -0.504686355591 -0.000112202587902 0.01\n",
      "149 150 -0.836894929409 -0.000112202587902 0.01\n",
      "149 200 -1.09835195541 -0.000112202587902 0.01\n",
      "149 250 -0.672937750816 -0.000112202587902 0.01\n",
      "149 300 -0.991402447224 -0.000112202587902 0.01\n",
      "149 350 -0.922642588615 -0.000112202587902 0.01\n",
      "149 400 -0.629194438457 -0.000112202587902 0.01\n",
      "149 450 -1.04556691647 -0.000112202587902 0.01\n",
      "149 500 -0.690356314182 -0.000112202587902 0.01\n",
      "149 550 -0.439627617598 -0.000112202587902 0.01\n",
      "valid_acc 79.435\n",
      "150 0 -0.737348258495 -0.000112202587902 0.01\n",
      "150 50 -0.85100966692 -0.000112202587902 0.01\n",
      "150 100 -0.567655086517 -0.000112202587902 0.01\n",
      "150 150 -0.611098229885 -0.000112202587902 0.01\n",
      "150 200 -0.606487929821 -0.000112202587902 0.01\n",
      "150 250 -0.479844927788 -0.000112202587902 0.01\n",
      "150 300 -0.597124576569 -0.000112202587902 0.01\n",
      "150 350 -0.568902254105 -0.000112202587902 0.01\n",
      "150 400 -0.946653962135 -0.000112202587902 0.01\n",
      "150 450 -0.820637285709 -0.000112202587902 0.01\n",
      "150 500 -0.509285986423 -0.000112202587902 0.01\n",
      "150 550 -1.0791426897 -0.000112202587902 0.01\n",
      "valid_acc 75.63666666666666\n",
      "151 0 -0.750601530075 -0.000112202587902 0.01\n",
      "151 50 -0.6619617939 -0.000112202587902 0.01\n",
      "151 100 -0.83079713583 -0.000112202587902 0.01\n",
      "151 150 -0.745498716831 -0.000112202587902 0.01\n",
      "151 200 -0.614109873772 -0.000112202587902 0.01\n",
      "151 250 -0.870942890644 -0.000112202587902 0.01\n",
      "151 300 -0.940799772739 -0.000112202587902 0.01\n",
      "151 350 -0.58241891861 -0.000112202587902 0.01\n",
      "151 400 -0.502052128315 -0.000112202587902 0.01\n",
      "151 450 -0.50941836834 -0.000112202587902 0.01\n",
      "151 500 -0.557747244835 -0.000112202587902 0.01\n",
      "151 550 -0.752299129963 -0.000112202587902 0.01\n",
      "valid_acc 76.30166666666666\n",
      "152 0 -0.579822957516 -0.000112202587902 0.01\n",
      "152 50 -0.671418786049 -0.000112202587902 0.01\n",
      "152 100 -0.855031788349 -0.000112202587902 0.01\n",
      "152 150 -0.751622319221 -0.000112202587902 0.01\n",
      "152 200 -0.552434504032 -0.000112202587902 0.01\n",
      "152 250 -0.72067540884 -0.000112202587902 0.01\n",
      "152 300 -0.673415780067 -0.000112202587902 0.01\n",
      "152 350 -1.05180513859 -0.000112202587902 0.01\n",
      "152 400 -0.493171721697 -0.000112202587902 0.01\n",
      "152 450 -0.718301475048 -0.000112202587902 0.01\n",
      "152 500 -0.620470941067 -0.000112202587902 0.01\n",
      "152 550 -0.795261621475 -0.000112202587902 0.01\n",
      "valid_acc 74.21\n",
      "153 0 -0.88497877121 -0.000112202587902 0.01\n",
      "153 50 -0.838523805141 -0.000112202587902 0.01\n",
      "153 100 -0.582710921764 -0.000112202587902 0.01\n",
      "153 150 -0.648689210415 -0.000112202587902 0.01\n",
      "153 200 -0.619858503342 -0.000112202587902 0.01\n",
      "153 250 -0.489155650139 -0.000112202587902 0.01\n",
      "153 300 -0.730864465237 -0.000112202587902 0.01\n",
      "153 350 -0.587823390961 -0.000112202587902 0.01\n",
      "153 400 -0.548141956329 -0.000112202587902 0.01\n",
      "153 450 -0.527271330357 -0.000112202587902 0.01\n",
      "153 500 -0.673707962036 -0.000112202587902 0.01\n",
      "153 550 -0.74529671669 -0.000112202587902 0.01\n",
      "valid_acc 77.20833333333333\n",
      "154 0 -0.756152808666 -0.000112202587902 0.01\n",
      "154 50 -1.0651756525 -0.000112202587902 0.01\n",
      "154 100 -0.501105129719 -0.000112202587902 0.01\n",
      "154 150 -0.771107912064 -0.000112202587902 0.01\n",
      "154 200 -0.387158572674 -0.000112202587902 0.01\n",
      "154 250 -0.638424515724 -0.000112202587902 0.01\n",
      "154 300 -0.480996549129 -0.000112202587902 0.01\n",
      "154 350 -0.563021242619 -0.000112202587902 0.01\n",
      "154 400 -0.687149405479 -0.000112202587902 0.01\n",
      "154 450 -0.532817423344 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154 500 -1.0243396759 -0.000112202587902 0.01\n",
      "154 550 -0.616888523102 -0.000112202587902 0.01\n",
      "valid_acc 77.59666666666666\n",
      "155 0 -0.70385324955 -0.000112202587902 0.01\n",
      "155 50 -0.877524793148 -0.000112202587902 0.01\n",
      "155 100 -0.65965294838 -0.000112202587902 0.01\n",
      "155 150 -0.394708067179 -0.000112202587902 0.01\n",
      "155 200 -0.686869204044 -0.000112202587902 0.01\n",
      "155 250 -0.668205857277 -0.000112202587902 0.01\n",
      "155 300 -0.866693794727 -0.000112202587902 0.01\n",
      "155 350 -0.8450756073 -0.000112202587902 0.01\n",
      "155 400 -1.16948068142 -0.000112202587902 0.01\n",
      "155 450 -0.640466690063 -0.000112202587902 0.01\n",
      "155 500 -0.65563929081 -0.000112202587902 0.01\n",
      "155 550 -0.835797071457 -0.000112202587902 0.01\n",
      "valid_acc 75.7\n",
      "156 0 -0.716855943203 -0.000112202587902 0.01\n",
      "156 50 -0.701193869114 -0.000112202587902 0.01\n",
      "156 100 -0.912376344204 -0.000112202587902 0.01\n",
      "156 150 -0.592077255249 -0.000112202587902 0.01\n",
      "156 200 -0.828932821751 -0.000112202587902 0.01\n",
      "156 250 -0.468738108873 -0.000112202587902 0.01\n",
      "156 300 -0.658851027489 -0.000112202587902 0.01\n",
      "156 350 -0.559846401215 -0.000112202587902 0.01\n",
      "156 400 -0.813375174999 -0.000112202587902 0.01\n",
      "156 450 -0.700593650341 -0.000112202587902 0.01\n",
      "156 500 -0.704021453857 -0.000112202587902 0.01\n",
      "156 550 -0.633364319801 -0.000112202587902 0.01\n",
      "valid_acc 75.28166666666667\n",
      "157 0 -0.71922814846 -0.000112202587902 0.01\n",
      "157 50 -0.645666837692 -0.000112202587902 0.01\n",
      "157 100 -0.553853154182 -0.000112202587902 0.01\n",
      "157 150 -0.764657080173 -0.000112202587902 0.01\n",
      "157 200 -0.696858167648 -0.000112202587902 0.01\n",
      "157 250 -0.514384746552 -0.000112202587902 0.01\n",
      "157 300 -0.856146991253 -0.000112202587902 0.01\n",
      "157 350 -0.651671826839 -0.000112202587902 0.01\n",
      "157 400 -0.71048271656 -0.000112202587902 0.01\n",
      "157 450 -0.600733697414 -0.000112202587902 0.01\n",
      "157 500 -0.942526340485 -0.000112202587902 0.01\n",
      "157 550 -0.786560833454 -0.000112202587902 0.01\n",
      "valid_acc 78.39\n",
      "158 0 -0.5899477005 -0.000112202587902 0.01\n",
      "158 50 -0.772090375423 -0.000112202587902 0.01\n",
      "158 100 -0.54231184721 -0.000112202587902 0.01\n",
      "158 150 -0.898000478745 -0.000112202587902 0.01\n",
      "158 200 -0.723049998283 -0.000112202587902 0.01\n",
      "158 250 -0.646669626236 -0.000112202587902 0.01\n",
      "158 300 -0.703530073166 -0.000112202587902 0.01\n",
      "158 350 -0.66093313694 -0.000112202587902 0.01\n",
      "158 400 -0.440679401159 -0.000112202587902 0.01\n",
      "158 450 -0.57230246067 -0.000112202587902 0.01\n",
      "158 500 -0.510621011257 -0.000112202587902 0.01\n",
      "158 550 -0.75032889843 -0.000112202587902 0.01\n",
      "valid_acc 77.735\n",
      "159 0 -0.735409259796 -0.000112202587902 0.01\n",
      "159 50 -0.575468361378 -0.000112202587902 0.01\n",
      "159 100 -0.745266258717 -0.000112202587902 0.01\n",
      "159 150 -0.795945584774 -0.000112202587902 0.01\n",
      "159 200 -0.720888078213 -0.000112202587902 0.01\n",
      "159 250 -0.974457085133 -0.000112202587902 0.01\n",
      "159 300 -0.556890845299 -0.000112202587902 0.01\n",
      "159 350 -0.712973117828 -0.000112202587902 0.01\n",
      "159 400 -0.758594512939 -0.000112202587902 0.01\n",
      "159 450 -0.411463499069 -0.000112202587902 0.01\n",
      "159 500 -0.84689116478 -0.000112202587902 0.01\n",
      "159 550 -0.549467325211 -0.000112202587902 0.01\n",
      "valid_acc 77.095\n",
      "160 0 -0.842258930206 -0.000112202587902 0.01\n",
      "160 50 -0.640768349171 -0.000112202587902 0.01\n",
      "160 100 -0.629039287567 -0.000112202587902 0.01\n",
      "160 150 -0.581309378147 -0.000112202587902 0.01\n",
      "160 200 -0.811808347702 -0.000112202587902 0.01\n",
      "160 250 -0.663685083389 -0.000112202587902 0.01\n",
      "160 300 -0.919935286045 -0.000112202587902 0.01\n",
      "160 350 -0.769691705704 -0.000112202587902 0.01\n",
      "160 400 -0.786519825459 -0.000112202587902 0.01\n",
      "160 450 -0.529525160789 -0.000112202587902 0.01\n",
      "160 500 -0.704061448574 -0.000112202587902 0.01\n",
      "160 550 -0.666388571262 -0.000112202587902 0.01\n",
      "valid_acc 75.10166666666666\n",
      "161 0 -0.996314823627 -0.000112202587902 0.01\n",
      "161 50 -0.737120270729 -0.000112202587902 0.01\n",
      "161 100 -0.667056024075 -0.000112202587902 0.01\n",
      "161 150 -0.819922089577 -0.000112202587902 0.01\n",
      "161 200 -1.06387484074 -0.000112202587902 0.01\n",
      "161 250 -0.563860118389 -0.000112202587902 0.01\n",
      "161 300 -0.80318146944 -0.000112202587902 0.01\n",
      "161 350 -0.803940415382 -0.000112202587902 0.01\n",
      "161 400 -0.728448569775 -0.000112202587902 0.01\n",
      "161 450 -1.15039050579 -0.000112202587902 0.01\n",
      "161 500 -0.743718326092 -0.000112202587902 0.01\n",
      "161 550 -0.650290727615 -0.000112202587902 0.01\n",
      "valid_acc 71.55166666666666\n",
      "162 0 -0.74105399847 -0.000112202587902 0.01\n",
      "162 50 -0.994006812572 -0.000112202587902 0.01\n",
      "162 100 -0.909494936466 -0.000112202587902 0.01\n",
      "162 150 -0.931774139404 -0.000112202587902 0.01\n",
      "162 200 -0.680883169174 -0.000112202587902 0.01\n",
      "162 250 -0.792290508747 -0.000112202587902 0.01\n",
      "162 300 -0.905111372471 -0.000112202587902 0.01\n",
      "162 350 -0.634376347065 -0.000112202587902 0.01\n",
      "162 400 -0.358484715223 -0.000112202587902 0.01\n",
      "162 450 -0.659235358238 -0.000112202587902 0.01\n",
      "162 500 -0.892122030258 -0.000112202587902 0.01\n",
      "162 550 -0.662239849567 -0.000112202587902 0.01\n",
      "valid_acc 69.42333333333333\n",
      "163 0 -0.829243004322 -0.000112202587902 0.01\n",
      "163 50 -0.576301336288 -0.000112202587902 0.01\n",
      "163 100 -0.964266180992 -0.000112202587902 0.01\n",
      "163 150 -1.00511419773 -0.000112202587902 0.01\n",
      "163 200 -0.887507796288 -0.000112202587902 0.01\n",
      "163 250 -1.00667512417 -0.000112202587902 0.01\n",
      "163 300 -0.544312775135 -0.000112202587902 0.01\n",
      "163 350 -0.917534649372 -0.000112202587902 0.01\n",
      "163 400 -0.831970512867 -0.000112202587902 0.01\n",
      "163 450 -0.77498704195 -0.000112202587902 0.01\n",
      "163 500 -0.774760007858 -0.000112202587902 0.01\n",
      "163 550 -0.657340228558 -0.000112202587902 0.01\n",
      "valid_acc 73.00333333333333\n",
      "164 0 -0.786154687405 -0.000112202587902 0.01\n",
      "164 50 -0.681204617023 -0.000112202587902 0.01\n",
      "164 100 -0.867795228958 -0.000112202587902 0.01\n",
      "164 150 -0.687260985374 -0.000112202587902 0.01\n",
      "164 200 -1.32609391212 -0.000112202587902 0.01\n",
      "164 250 -0.390367895365 -0.000112202587902 0.01\n",
      "164 300 -0.79888343811 -0.000112202587902 0.01\n",
      "164 350 -0.726226091385 -0.000112202587902 0.01\n",
      "164 400 -0.764637351036 -0.000112202587902 0.01\n",
      "164 450 -0.873883426189 -0.000112202587902 0.01\n",
      "164 500 -0.719705045223 -0.000112202587902 0.01\n",
      "164 550 -0.650030493736 -0.000112202587902 0.01\n",
      "valid_acc 76.42166666666667\n",
      "165 0 -0.664033889771 -0.000112202587902 0.01\n",
      "165 50 -1.03307366371 -0.000112202587902 0.01\n",
      "165 100 -0.851596593857 -0.000112202587902 0.01\n",
      "165 150 -0.733798742294 -0.000112202587902 0.01\n",
      "165 200 -0.660763919353 -0.000112202587902 0.01\n",
      "165 250 -1.13471186161 -0.000112202587902 0.01\n",
      "165 300 -0.826934337616 -0.000112202587902 0.01\n",
      "165 350 -0.564696252346 -0.000112202587902 0.01\n",
      "165 400 -0.639966547489 -0.000112202587902 0.01\n",
      "165 450 -0.648614704609 -0.000112202587902 0.01\n",
      "165 500 -0.727508187294 -0.000112202587902 0.01\n",
      "165 550 -0.733191430569 -0.000112202587902 0.01\n",
      "valid_acc 78.48166666666667\n",
      "166 0 -0.530423045158 -0.000112202587902 0.01\n",
      "166 50 -0.785626351833 -0.000112202587902 0.01\n",
      "166 100 -0.871476948261 -0.000112202587902 0.01\n",
      "166 150 -0.733493208885 -0.000112202587902 0.01\n",
      "166 200 -0.558017015457 -0.000112202587902 0.01\n",
      "166 250 -0.713623046875 -0.000112202587902 0.01\n",
      "166 300 -0.693112611771 -0.000112202587902 0.01\n",
      "166 350 -0.785476326942 -0.000112202587902 0.01\n",
      "166 400 -0.77663987875 -0.000112202587902 0.01\n",
      "166 450 -0.574614644051 -0.000112202587902 0.01\n",
      "166 500 -0.736882388592 -0.000112202587902 0.01\n",
      "166 550 -0.802924215794 -0.000112202587902 0.01\n",
      "valid_acc 78.80666666666667\n",
      "167 0 -0.712283611298 -0.000112202587902 0.01\n",
      "167 50 -0.577446043491 -0.000112202587902 0.01\n",
      "167 100 -0.677769482136 -0.000112202587902 0.01\n",
      "167 150 -0.586891531944 -0.000112202587902 0.01\n",
      "167 200 -0.614268541336 -0.000112202587902 0.01\n",
      "167 250 -0.728203833103 -0.000112202587902 0.01\n",
      "167 300 -1.01208508015 -0.000112202587902 0.01\n",
      "167 350 -0.763237297535 -0.000112202587902 0.01\n",
      "167 400 -0.695292830467 -0.000112202587902 0.01\n",
      "167 450 -0.640557467937 -0.000112202587902 0.01\n",
      "167 500 -0.738388895988 -0.000112202587902 0.01\n",
      "167 550 -0.923489093781 -0.000112202587902 0.01\n",
      "valid_acc 76.32833333333333\n",
      "168 0 -0.589851617813 -0.000112202587902 0.01\n",
      "168 50 -0.853808760643 -0.000112202587902 0.01\n",
      "168 100 -0.682683885098 -0.000112202587902 0.01\n",
      "168 150 -0.863387465477 -0.000112202587902 0.01\n",
      "168 200 -0.636669874191 -0.000112202587902 0.01\n",
      "168 250 -0.84277176857 -0.000112202587902 0.01\n",
      "168 300 -1.06730699539 -0.000112202587902 0.01\n",
      "168 350 -0.570159971714 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "168 400 -0.483890920877 -0.000112202587902 0.01\n",
      "168 450 -0.659655928612 -0.000112202587902 0.01\n",
      "168 500 -0.710434019566 -0.000112202587902 0.01\n",
      "168 550 -0.841409742832 -0.000112202587902 0.01\n",
      "valid_acc 75.36166666666666\n",
      "169 0 -0.823440492153 -0.000112202587902 0.01\n",
      "169 50 -0.636347770691 -0.000112202587902 0.01\n",
      "169 100 -0.935267806053 -0.000112202587902 0.01\n",
      "169 150 -0.516421616077 -0.000112202587902 0.01\n",
      "169 200 -0.63932710886 -0.000112202587902 0.01\n",
      "169 250 -0.594737350941 -0.000112202587902 0.01\n",
      "169 300 -0.781294226646 -0.000112202587902 0.01\n",
      "169 350 -0.607886254787 -0.000112202587902 0.01\n",
      "169 400 -0.72407323122 -0.000112202587902 0.01\n",
      "169 450 -0.595680177212 -0.000112202587902 0.01\n",
      "169 500 -0.905075013638 -0.000112202587902 0.01\n",
      "169 550 -1.11933076382 -0.000112202587902 0.01\n",
      "valid_acc 79.32666666666667\n",
      "170 0 -0.948545217514 -0.000112202587902 0.01\n",
      "170 50 -0.664179027081 -0.000112202587902 0.01\n",
      "170 100 -0.700203180313 -0.000112202587902 0.01\n",
      "170 150 -0.650559365749 -0.000112202587902 0.01\n",
      "170 200 -0.625169813633 -0.000112202587902 0.01\n",
      "170 250 -0.71746057272 -0.000112202587902 0.01\n",
      "170 300 -0.671451032162 -0.000112202587902 0.01\n",
      "170 350 -0.614441931248 -0.000112202587902 0.01\n",
      "170 400 -0.56221216917 -0.000112202587902 0.01\n",
      "170 450 -0.764700770378 -0.000112202587902 0.01\n",
      "170 500 -0.878042459488 -0.000112202587902 0.01\n",
      "170 550 -0.831505596638 -0.000112202587902 0.01\n",
      "valid_acc 71.61166666666666\n",
      "171 0 -0.437885820866 -0.000112202587902 0.01\n",
      "171 50 -0.849910140038 -0.000112202587902 0.01\n",
      "171 100 -0.459914803505 -0.000112202587902 0.01\n",
      "171 150 -0.69784361124 -0.000112202587902 0.01\n",
      "171 200 -0.637114167213 -0.000112202587902 0.01\n",
      "171 250 -0.599875152111 -0.000112202587902 0.01\n",
      "171 300 -0.534506618977 -0.000112202587902 0.01\n",
      "171 350 -0.687522530556 -0.000112202587902 0.01\n",
      "171 400 -0.947978079319 -0.000112202587902 0.01\n",
      "171 450 -0.889076530933 -0.000112202587902 0.01\n",
      "171 500 -0.690076768398 -0.000112202587902 0.01\n",
      "171 550 -0.78488945961 -0.000112202587902 0.01\n",
      "valid_acc 77.835\n",
      "172 0 -0.870511114597 -0.000112202587902 0.01\n",
      "172 50 -0.821605622768 -0.000112202587902 0.01\n",
      "172 100 -0.679249346256 -0.000112202587902 0.01\n",
      "172 150 -0.965766727924 -0.000112202587902 0.01\n",
      "172 200 -0.893926382065 -0.000112202587902 0.01\n",
      "172 250 -1.01040279865 -0.000112202587902 0.01\n",
      "172 300 -0.954954981804 -0.000112202587902 0.01\n",
      "172 350 -0.61105376482 -0.000112202587902 0.01\n",
      "172 400 -0.98184210062 -0.000112202587902 0.01\n",
      "172 450 -0.57707631588 -0.000112202587902 0.01\n",
      "172 500 -0.827076196671 -0.000112202587902 0.01\n",
      "172 550 -0.507540106773 -0.000112202587902 0.01\n",
      "valid_acc 76.20166666666667\n",
      "173 0 -0.788337647915 -0.000112202587902 0.01\n",
      "173 50 -0.986807465553 -0.000112202587902 0.01\n",
      "173 100 -0.610309243202 -0.000112202587902 0.01\n",
      "173 150 -0.926742255688 -0.000112202587902 0.01\n",
      "173 200 -0.654927313328 -0.000112202587902 0.01\n",
      "173 250 -0.902160108089 -0.000112202587902 0.01\n",
      "173 300 -0.689248144627 -0.000112202587902 0.01\n",
      "173 350 -0.748650610447 -0.000112202587902 0.01\n",
      "173 400 -1.0348893404 -0.000112202587902 0.01\n",
      "173 450 -0.703613758087 -0.000112202587902 0.01\n",
      "173 500 -0.760938823223 -0.000112202587902 0.01\n",
      "173 550 -1.01675236225 -0.000112202587902 0.01\n",
      "valid_acc 79.00166666666667\n",
      "174 0 -0.428322404623 -0.000112202587902 0.01\n",
      "174 50 -0.752609312534 -0.000112202587902 0.01\n",
      "174 100 -0.674888134003 -0.000112202587902 0.01\n",
      "174 150 -0.777113497257 -0.000112202587902 0.01\n",
      "174 200 -0.755485296249 -0.000112202587902 0.01\n",
      "174 250 -0.753409445286 -0.000112202587902 0.01\n",
      "174 300 -0.797191262245 -0.000112202587902 0.01\n",
      "174 350 -0.573633670807 -0.000112202587902 0.01\n",
      "174 400 -0.961605429649 -0.000112202587902 0.01\n",
      "174 450 -0.690105736256 -0.000112202587902 0.01\n",
      "174 500 -0.865858435631 -0.000112202587902 0.01\n",
      "174 550 -0.579541027546 -0.000112202587902 0.01\n",
      "valid_acc 74.35000000000001\n",
      "175 0 -0.644905626774 -0.000112202587902 0.01\n",
      "175 50 -0.858720302582 -0.000112202587902 0.01\n",
      "175 100 -0.804567813873 -0.000112202587902 0.01\n",
      "175 150 -0.879516303539 -0.000112202587902 0.01\n",
      "175 200 -1.04178988934 -0.000112202587902 0.01\n",
      "175 250 -0.933876216412 -0.000112202587902 0.01\n",
      "175 300 -0.646685004234 -0.000112202587902 0.01\n",
      "175 350 -0.790386915207 -0.000112202587902 0.01\n",
      "175 400 -0.787897348404 -0.000112202587902 0.01\n",
      "175 450 -0.679827511311 -0.000112202587902 0.01\n",
      "175 500 -0.47071570158 -0.000112202587902 0.01\n",
      "175 550 -1.03240573406 -0.000112202587902 0.01\n",
      "valid_acc 73.46833333333333\n",
      "176 0 -1.1735855341 -0.000112202587902 0.01\n",
      "176 50 -0.66664403677 -0.000112202587902 0.01\n",
      "176 100 -0.454304039478 -0.000112202587902 0.01\n",
      "176 150 -0.833831489086 -0.000112202587902 0.01\n",
      "176 200 -0.509228467941 -0.000112202587902 0.01\n",
      "176 250 -0.849341332912 -0.000112202587902 0.01\n",
      "176 300 -1.00727045536 -0.000112202587902 0.01\n",
      "176 350 -1.03525960445 -0.000112202587902 0.01\n",
      "176 400 -0.769160866737 -0.000112202587902 0.01\n",
      "176 450 -1.15585899353 -0.000112202587902 0.01\n",
      "176 500 -0.840313494205 -0.000112202587902 0.01\n",
      "176 550 -0.664806365967 -0.000112202587902 0.01\n",
      "valid_acc 77.445\n",
      "177 0 -0.652428507805 -0.000112202587902 0.01\n",
      "177 50 -0.820642113686 -0.000112202587902 0.01\n",
      "177 100 -0.657019019127 -0.000112202587902 0.01\n",
      "177 150 -1.01358520985 -0.000112202587902 0.01\n",
      "177 200 -0.914501786232 -0.000112202587902 0.01\n",
      "177 250 -0.647815525532 -0.000112202587902 0.01\n",
      "177 300 -0.810702562332 -0.000112202587902 0.01\n",
      "177 350 -0.710966348648 -0.000112202587902 0.01\n",
      "177 400 -0.338999986649 -0.000112202587902 0.01\n",
      "177 450 -0.85605943203 -0.000112202587902 0.01\n",
      "177 500 -0.784196317196 -0.000112202587902 0.01\n",
      "177 550 -0.805883765221 -0.000112202587902 0.01\n",
      "valid_acc 77.56666666666666\n",
      "178 0 -0.92915391922 -0.000112202587902 0.01\n",
      "178 50 -0.639012634754 -0.000112202587902 0.01\n",
      "178 100 -0.82165825367 -0.000112202587902 0.01\n",
      "178 150 -0.75810188055 -0.000112202587902 0.01\n",
      "178 200 -0.81342369318 -0.000112202587902 0.01\n",
      "178 250 -0.837115049362 -0.000112202587902 0.01\n",
      "178 300 -0.532234430313 -0.000112202587902 0.01\n",
      "178 350 -1.23437166214 -0.000112202587902 0.01\n",
      "178 400 -0.757854938507 -0.000112202587902 0.01\n",
      "178 450 -0.998579323292 -0.000112202587902 0.01\n",
      "178 500 -0.771080851555 -0.000112202587902 0.01\n",
      "178 550 -0.566104829311 -0.000112202587902 0.01\n",
      "valid_acc 75.45166666666667\n",
      "179 0 -0.744031310081 -0.000112202587902 0.01\n",
      "179 50 -0.572698950768 -0.000112202587902 0.01\n",
      "179 100 -0.671864330769 -0.000112202587902 0.01\n",
      "179 150 -0.819874107838 -0.000112202587902 0.01\n",
      "179 200 -0.814399778843 -0.000112202587902 0.01\n",
      "179 250 -0.614530205727 -0.000112202587902 0.01\n",
      "179 300 -0.747786879539 -0.000112202587902 0.01\n",
      "179 350 -0.807754993439 -0.000112202587902 0.01\n",
      "179 400 -0.845492124557 -0.000112202587902 0.01\n",
      "179 450 -0.74736648798 -0.000112202587902 0.01\n",
      "179 500 -1.00571906567 -0.000112202587902 0.01\n",
      "179 550 -0.825314283371 -0.000112202587902 0.01\n",
      "valid_acc 74.41333333333333\n",
      "180 0 -0.63140052557 -0.000112202587902 0.01\n",
      "180 50 -0.897233784199 -0.000112202587902 0.01\n",
      "180 100 -0.740715324879 -0.000112202587902 0.01\n",
      "180 150 -0.880245923996 -0.000112202587902 0.01\n",
      "180 200 -0.605027377605 -0.000112202587902 0.01\n",
      "180 250 -0.679976284504 -0.000112202587902 0.01\n",
      "180 300 -0.873774111271 -0.000112202587902 0.01\n",
      "180 350 -0.713429927826 -0.000112202587902 0.01\n",
      "180 400 -0.854054272175 -0.000112202587902 0.01\n",
      "180 450 -0.628525912762 -0.000112202587902 0.01\n",
      "180 500 -0.687702775002 -0.000112202587902 0.01\n",
      "180 550 -1.00005686283 -0.000112202587902 0.01\n",
      "valid_acc 74.74333333333333\n",
      "181 0 -1.15758132935 -0.000112202587902 0.01\n",
      "181 50 -1.37777268887 -0.000112202587902 0.01\n",
      "181 100 -0.618442952633 -0.000112202587902 0.01\n",
      "181 150 -1.15304803848 -0.000112202587902 0.01\n",
      "181 200 -0.541656851768 -0.000112202587902 0.01\n",
      "181 250 -0.579326450825 -0.000112202587902 0.01\n",
      "181 300 -0.696491956711 -0.000112202587902 0.01\n",
      "181 350 -0.950471282005 -0.000112202587902 0.01\n",
      "181 400 -0.681918203831 -0.000112202587902 0.01\n",
      "181 450 -0.494378089905 -0.000112202587902 0.01\n",
      "181 500 -1.13990998268 -0.000112202587902 0.01\n",
      "181 550 -0.611656665802 -0.000112202587902 0.01\n",
      "valid_acc 74.72333333333333\n",
      "182 0 -0.853476107121 -0.000112202587902 0.01\n",
      "182 50 -0.884681761265 -0.000112202587902 0.01\n",
      "182 100 -0.828070700169 -0.000112202587902 0.01\n",
      "182 150 -0.57980453968 -0.000112202587902 0.01\n",
      "182 200 -0.929689645767 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "182 250 -0.755557179451 -0.000112202587902 0.01\n",
      "182 300 -0.835758507252 -0.000112202587902 0.01\n",
      "182 350 -1.27157080173 -0.000112202587902 0.01\n",
      "182 400 -0.800491034985 -0.000112202587902 0.01\n",
      "182 450 -0.846607148647 -0.000112202587902 0.01\n",
      "182 500 -0.827574789524 -0.000112202587902 0.01\n",
      "182 550 -0.702320337296 -0.000112202587902 0.01\n",
      "valid_acc 76.19\n",
      "183 0 -0.745704293251 -0.000112202587902 0.01\n",
      "183 50 -0.799405753613 -0.000112202587902 0.01\n",
      "183 100 -0.809553146362 -0.000112202587902 0.01\n",
      "183 150 -0.530694782734 -0.000112202587902 0.01\n",
      "183 200 -0.465220212936 -0.000112202587902 0.01\n",
      "183 250 -0.61411100626 -0.000112202587902 0.01\n",
      "183 300 -0.438476145267 -0.000112202587902 0.01\n",
      "183 350 -0.831314563751 -0.000112202587902 0.01\n",
      "183 400 -0.683604717255 -0.000112202587902 0.01\n",
      "183 450 -0.907364666462 -0.000112202587902 0.01\n",
      "183 500 -0.917754411697 -0.000112202587902 0.01\n",
      "183 550 -0.760115504265 -0.000112202587902 0.01\n",
      "valid_acc 77.02\n",
      "184 0 -0.911224663258 -0.000112202587902 0.01\n",
      "184 50 -0.873087823391 -0.000112202587902 0.01\n",
      "184 100 -0.660494446754 -0.000112202587902 0.01\n",
      "184 150 -0.616361916065 -0.000112202587902 0.01\n",
      "184 200 -0.628170073032 -0.000112202587902 0.01\n",
      "184 250 -0.448942840099 -0.000112202587902 0.01\n",
      "184 300 -0.956964850426 -0.000112202587902 0.01\n",
      "184 350 -0.893597364426 -0.000112202587902 0.01\n",
      "184 400 -0.733285546303 -0.000112202587902 0.01\n",
      "184 450 -0.99358445406 -0.000112202587902 0.01\n",
      "184 500 -1.09078550339 -0.000112202587902 0.01\n",
      "184 550 -0.906219959259 -0.000112202587902 0.01\n",
      "valid_acc 72.93166666666666\n",
      "185 0 -1.19370901585 -0.000112202587902 0.01\n",
      "185 50 -0.746908962727 -0.000112202587902 0.01\n",
      "185 100 -0.557069301605 -0.000112202587902 0.01\n",
      "185 150 -0.786794185638 -0.000112202587902 0.01\n",
      "185 200 -0.77625143528 -0.000112202587902 0.01\n",
      "185 250 -0.666991710663 -0.000112202587902 0.01\n",
      "185 300 -0.517755866051 -0.000112202587902 0.01\n",
      "185 350 -1.25640881062 -0.000112202587902 0.01\n",
      "185 400 -0.468123555183 -0.000112202587902 0.01\n",
      "185 450 -0.766757190228 -0.000112202587902 0.01\n",
      "185 500 -0.468078166246 -0.000112202587902 0.01\n",
      "185 550 -0.719305038452 -0.000112202587902 0.01\n",
      "valid_acc 72.405\n",
      "186 0 -0.727225542068 -0.000112202587902 0.01\n",
      "186 50 -0.942995071411 -0.000112202587902 0.01\n",
      "186 100 -0.810916423798 -0.000112202587902 0.01\n",
      "186 150 -0.68698579073 -0.000112202587902 0.01\n",
      "186 200 -0.711621761322 -0.000112202587902 0.01\n",
      "186 250 -0.586007833481 -0.000112202587902 0.01\n",
      "186 300 -0.633802473545 -0.000112202587902 0.01\n",
      "186 350 -0.801292359829 -0.000112202587902 0.01\n",
      "186 400 -0.938381671906 -0.000112202587902 0.01\n",
      "186 450 -0.571857333183 -0.000112202587902 0.01\n",
      "186 500 -0.779840946198 -0.000112202587902 0.01\n",
      "186 550 -0.866857349873 -0.000112202587902 0.01\n",
      "valid_acc 77.645\n",
      "187 0 -0.985494017601 -0.000112202587902 0.01\n",
      "187 50 -0.770561277866 -0.000112202587902 0.01\n",
      "187 100 -0.560209751129 -0.000112202587902 0.01\n",
      "187 150 -0.660541236401 -0.000112202587902 0.01\n",
      "187 200 -1.1744184494 -0.000112202587902 0.01\n",
      "187 250 -0.964715600014 -0.000112202587902 0.01\n",
      "187 300 -0.668464481831 -0.000112202587902 0.01\n",
      "187 350 -0.812418282032 -0.000112202587902 0.01\n",
      "187 400 -0.743632555008 -0.000112202587902 0.01\n",
      "187 450 -0.890712916851 -0.000112202587902 0.01\n",
      "187 500 -1.02460217476 -0.000112202587902 0.01\n",
      "187 550 -0.486567646265 -0.000112202587902 0.01\n",
      "valid_acc 74.67833333333334\n",
      "188 0 -0.52761054039 -0.000112202587902 0.01\n",
      "188 50 -0.637801647186 -0.000112202587902 0.01\n",
      "188 100 -0.681341648102 -0.000112202587902 0.01\n",
      "188 150 -0.859502315521 -0.000112202587902 0.01\n",
      "188 200 -0.605232954025 -0.000112202587902 0.01\n",
      "188 250 -0.73886269331 -0.000112202587902 0.01\n",
      "188 300 -0.7678642869 -0.000112202587902 0.01\n",
      "188 350 -0.850797891617 -0.000112202587902 0.01\n",
      "188 400 -0.891497969627 -0.000112202587902 0.01\n",
      "188 450 -0.785537421703 -0.000112202587902 0.01\n",
      "188 500 -1.07052373886 -0.000112202587902 0.01\n",
      "188 550 -0.986575782299 -0.000112202587902 0.01\n",
      "valid_acc 76.25666666666666\n",
      "189 0 -0.72437363863 -0.000112202587902 0.01\n",
      "189 50 -0.874730527401 -0.000112202587902 0.01\n",
      "189 100 -1.11224901676 -0.000112202587902 0.01\n",
      "189 150 -0.890079677105 -0.000112202587902 0.01\n",
      "189 200 -0.687028825283 -0.000112202587902 0.01\n",
      "189 250 -0.798446416855 -0.000112202587902 0.01\n",
      "189 300 -0.827215492725 -0.000112202587902 0.01\n",
      "189 350 -0.851258456707 -0.000112202587902 0.01\n",
      "189 400 -0.588971555233 -0.000112202587902 0.01\n",
      "189 450 -0.902353584766 -0.000112202587902 0.01\n",
      "189 500 -0.938582897186 -0.000112202587902 0.01\n",
      "189 550 -0.699125051498 -0.000112202587902 0.01\n",
      "valid_acc 79.07499999999999\n",
      "190 0 -0.533109903336 -0.000112202587902 0.01\n",
      "190 50 -0.795296013355 -0.000112202587902 0.01\n",
      "190 100 -1.38594269753 -0.000112202587902 0.01\n",
      "190 150 -0.823849856853 -0.000112202587902 0.01\n",
      "190 200 -0.803561151028 -0.000112202587902 0.01\n",
      "190 250 -0.868752002716 -0.000112202587902 0.01\n",
      "190 300 -0.83539968729 -0.000112202587902 0.01\n",
      "190 350 -0.903010427952 -0.000112202587902 0.01\n",
      "190 400 -0.873146831989 -0.000112202587902 0.01\n",
      "190 450 -0.675176978111 -0.000112202587902 0.01\n",
      "190 500 -0.84221726656 -0.000112202587902 0.01\n",
      "190 550 -0.89248251915 -0.000112202587902 0.01\n",
      "valid_acc 78.70333333333333\n",
      "191 0 -0.588030159473 -0.000112202587902 0.01\n",
      "191 50 -0.843291461468 -0.000112202587902 0.01\n",
      "191 100 -0.730108261108 -0.000112202587902 0.01\n",
      "191 150 -0.86984372139 -0.000112202587902 0.01\n",
      "191 200 -0.583556771278 -0.000112202587902 0.01\n",
      "191 250 -0.629665434361 -0.000112202587902 0.01\n",
      "191 300 -0.60630106926 -0.000112202587902 0.01\n",
      "191 350 -0.656685709953 -0.000112202587902 0.01\n",
      "191 400 -1.00495862961 -0.000112202587902 0.01\n",
      "191 450 -0.609754621983 -0.000112202587902 0.01\n",
      "191 500 -0.584072113037 -0.000112202587902 0.01\n",
      "191 550 -1.110247612 -0.000112202587902 0.01\n",
      "valid_acc 72.50333333333333\n",
      "192 0 -1.22020697594 -0.000112202587902 0.01\n",
      "192 50 -0.914474010468 -0.000112202587902 0.01\n",
      "192 100 -0.706723630428 -0.000112202587902 0.01\n",
      "192 150 -0.764071583748 -0.000112202587902 0.01\n",
      "192 200 -0.670996844769 -0.000112202587902 0.01\n",
      "192 250 -0.604303002357 -0.000112202587902 0.01\n",
      "192 300 -0.79419118166 -0.000112202587902 0.01\n",
      "192 350 -0.668502986431 -0.000112202587902 0.01\n",
      "192 400 -0.745801329613 -0.000112202587902 0.01\n",
      "192 450 -0.60099375248 -0.000112202587902 0.01\n",
      "192 500 -0.609682679176 -0.000112202587902 0.01\n",
      "192 550 -0.848789989948 -0.000112202587902 0.01\n",
      "valid_acc 75.18333333333334\n",
      "193 0 -0.727678358555 -0.000112202587902 0.01\n",
      "193 50 -0.8395845294 -0.000112202587902 0.01\n",
      "193 100 -0.644126951694 -0.000112202587902 0.01\n",
      "193 150 -0.717888116837 -0.000112202587902 0.01\n",
      "193 200 -0.933437824249 -0.000112202587902 0.01\n",
      "193 250 -0.575593352318 -0.000112202587902 0.01\n",
      "193 300 -0.795934081078 -0.000112202587902 0.01\n",
      "193 350 -0.767126142979 -0.000112202587902 0.01\n",
      "193 400 -0.64449518919 -0.000112202587902 0.01\n",
      "193 450 -0.674536883831 -0.000112202587902 0.01\n",
      "193 500 -0.873103320599 -0.000112202587902 0.01\n",
      "193 550 -0.585414171219 -0.000112202587902 0.01\n",
      "valid_acc 76.82166666666667\n",
      "194 0 -0.865878283978 -0.000112202587902 0.01\n",
      "194 50 -0.541298627853 -0.000112202587902 0.01\n",
      "194 100 -0.597261369228 -0.000112202587902 0.01\n",
      "194 150 -0.756711900234 -0.000112202587902 0.01\n",
      "194 200 -0.576998531818 -0.000112202587902 0.01\n",
      "194 250 -0.607061266899 -0.000112202587902 0.01\n",
      "194 300 -0.829898059368 -0.000112202587902 0.01\n",
      "194 350 -1.18923795223 -0.000112202587902 0.01\n",
      "194 400 -0.853372752666 -0.000112202587902 0.01\n",
      "194 450 -0.584587574005 -0.000112202587902 0.01\n",
      "194 500 -0.957731723785 -0.000112202587902 0.01\n",
      "194 550 -0.721022188663 -0.000112202587902 0.01\n",
      "valid_acc 67.98666666666666\n",
      "195 0 -1.67779743671 -0.000112202587902 0.01\n",
      "195 50 -0.621551275253 -0.000112202587902 0.01\n",
      "195 100 -0.679331660271 -0.000112202587902 0.01\n",
      "195 150 -0.492512047291 -0.000112202587902 0.01\n",
      "195 200 -0.610013484955 -0.000112202587902 0.01\n",
      "195 250 -0.523898720741 -0.000112202587902 0.01\n",
      "195 300 -0.476282656193 -0.000112202587902 0.01\n",
      "195 350 -0.641362309456 -0.000112202587902 0.01\n",
      "195 400 -0.929358422756 -0.000112202587902 0.01\n",
      "195 450 -0.680322408676 -0.000112202587902 0.01\n",
      "195 500 -0.609694898129 -0.000112202587902 0.01\n",
      "195 550 -0.616348743439 -0.000112202587902 0.01\n",
      "valid_acc 77.22666666666666\n",
      "196 0 -0.49802261591 -0.000112202587902 0.01\n",
      "196 50 -0.638968586922 -0.000112202587902 0.01\n",
      "196 100 -0.793980419636 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196 150 -0.768591284752 -0.000112202587902 0.01\n",
      "196 200 -0.483899772167 -0.000112202587902 0.01\n",
      "196 250 -0.650021135807 -0.000112202587902 0.01\n",
      "196 300 -0.586133301258 -0.000112202587902 0.01\n",
      "196 350 -0.760885298252 -0.000112202587902 0.01\n",
      "196 400 -0.495225310326 -0.000112202587902 0.01\n",
      "196 450 -0.90964615345 -0.000112202587902 0.01\n",
      "196 500 -0.926032662392 -0.000112202587902 0.01\n",
      "196 550 -0.727082431316 -0.000112202587902 0.01\n",
      "valid_acc 75.26166666666667\n",
      "197 0 -0.846207380295 -0.000112202587902 0.01\n",
      "197 50 -0.503271043301 -0.000112202587902 0.01\n",
      "197 100 -0.897483229637 -0.000112202587902 0.01\n",
      "197 150 -0.893833875656 -0.000112202587902 0.01\n",
      "197 200 -0.666286766529 -0.000112202587902 0.01\n",
      "197 250 -0.823375999928 -0.000112202587902 0.01\n",
      "197 300 -0.487695008516 -0.000112202587902 0.01\n",
      "197 350 -0.544250428677 -0.000112202587902 0.01\n",
      "197 400 -0.950803041458 -0.000112202587902 0.01\n",
      "197 450 -0.694464504719 -0.000112202587902 0.01\n",
      "197 500 -0.724992036819 -0.000112202587902 0.01\n",
      "197 550 -0.731771409512 -0.000112202587902 0.01\n",
      "valid_acc 79.77666666666666\n",
      "198 0 -0.608470082283 -0.000112202587902 0.01\n",
      "198 50 -0.75468057394 -0.000112202587902 0.01\n",
      "198 100 -0.541338026524 -0.000112202587902 0.01\n",
      "198 150 -0.448107749224 -0.000112202587902 0.01\n",
      "198 200 -0.522268772125 -0.000112202587902 0.01\n",
      "198 250 -0.736344575882 -0.000112202587902 0.01\n",
      "198 300 -0.966320931911 -0.000112202587902 0.01\n",
      "198 350 -0.809618294239 -0.000112202587902 0.01\n",
      "198 400 -0.603393018246 -0.000112202587902 0.01\n",
      "198 450 -0.590720415115 -0.000112202587902 0.01\n",
      "198 500 -0.629380166531 -0.000112202587902 0.01\n",
      "198 550 -0.326278924942 -0.000112202587902 0.01\n",
      "valid_acc 79.45333333333333\n",
      "199 0 -0.557485759258 -0.000112202587902 0.01\n",
      "199 50 -0.509172260761 -0.000112202587902 0.01\n",
      "199 100 -0.508538663387 -0.000112202587902 0.01\n",
      "199 150 -0.419241338968 -0.000112202587902 0.01\n",
      "199 200 -0.644526302814 -0.000112202587902 0.01\n",
      "199 250 -0.822006285191 -0.000112202587902 0.01\n",
      "199 300 -0.607015371323 -0.000112202587902 0.01\n",
      "199 350 -0.552526652813 -0.000112202587902 0.01\n",
      "199 400 -0.639456987381 -0.000112202587902 0.01\n",
      "199 450 -0.831115961075 -0.000112202587902 0.01\n",
      "199 500 -0.842433989048 -0.000112202587902 0.01\n",
      "199 550 -0.975350439548 -0.000112202587902 0.01\n",
      "valid_acc 77.44\n",
      "200 0 -1.16972136497 -0.000112202587902 0.01\n",
      "200 50 -0.301721334457 -0.000112202587902 0.01\n",
      "200 100 -0.888821482658 -0.000112202587902 0.01\n",
      "200 150 -0.564786195755 -0.000112202587902 0.01\n",
      "200 200 -0.654654920101 -0.000112202587902 0.01\n",
      "200 250 -0.844348013401 -0.000112202587902 0.01\n",
      "200 300 -0.726266384125 -0.000112202587902 0.01\n",
      "200 350 -0.886387765408 -0.000112202587902 0.01\n",
      "200 400 -0.741970658302 -0.000112202587902 0.01\n",
      "200 450 -0.440601587296 -0.000112202587902 0.01\n",
      "200 500 -0.73577606678 -0.000112202587902 0.01\n",
      "200 550 -0.6512824893 -0.000112202587902 0.01\n",
      "valid_acc 76.22166666666666\n",
      "201 0 -0.849060058594 -0.000112202587902 0.01\n",
      "201 50 -0.837144136429 -0.000112202587902 0.01\n",
      "201 100 -0.636652171612 -0.000112202587902 0.01\n",
      "201 150 -0.70045030117 -0.000112202587902 0.01\n",
      "201 200 -0.488419651985 -0.000112202587902 0.01\n",
      "201 250 -0.704932034016 -0.000112202587902 0.01\n",
      "201 300 -0.993430197239 -0.000112202587902 0.01\n",
      "201 350 -0.867255926132 -0.000112202587902 0.01\n",
      "201 400 -0.540033280849 -0.000112202587902 0.01\n",
      "201 450 -0.593936264515 -0.000112202587902 0.01\n",
      "201 500 -0.481891751289 -0.000112202587902 0.01\n",
      "201 550 -0.835982084274 -0.000112202587902 0.01\n",
      "valid_acc 78.30166666666668\n",
      "202 0 -0.807720661163 -0.000112202587902 0.01\n",
      "202 50 -0.777973651886 -0.000112202587902 0.01\n",
      "202 100 -0.687476754189 -0.000112202587902 0.01\n",
      "202 150 -0.498494952917 -0.000112202587902 0.01\n",
      "202 200 -0.520627140999 -0.000112202587902 0.01\n",
      "202 250 -0.475602984428 -0.000112202587902 0.01\n",
      "202 300 -0.701787114143 -0.000112202587902 0.01\n",
      "202 350 -0.531908452511 -0.000112202587902 0.01\n",
      "202 400 -0.666620731354 -0.000112202587902 0.01\n",
      "202 450 -0.708121657372 -0.000112202587902 0.01\n",
      "202 500 -0.672228455544 -0.000112202587902 0.01\n",
      "202 550 -0.465208560228 -0.000112202587902 0.01\n",
      "valid_acc 77.88333333333334\n",
      "203 0 -0.915323317051 -0.000112202587902 0.01\n",
      "203 50 -0.538701295853 -0.000112202587902 0.01\n",
      "203 100 -0.389790296555 -0.000112202587902 0.01\n",
      "203 150 -1.10770106316 -0.000112202587902 0.01\n",
      "203 200 -0.621493935585 -0.000112202587902 0.01\n",
      "203 250 -0.561278045177 -0.000112202587902 0.01\n",
      "203 300 -0.68612486124 -0.000112202587902 0.01\n",
      "203 350 -0.488182842731 -0.000112202587902 0.01\n",
      "203 400 -0.98948264122 -0.000112202587902 0.01\n",
      "203 450 -0.875883281231 -0.000112202587902 0.01\n",
      "203 500 -0.93009698391 -0.000112202587902 0.01\n",
      "203 550 -0.441191017628 -0.000112202587902 0.01\n",
      "valid_acc 76.96666666666667\n",
      "204 0 -1.16778051853 -0.000112202587902 0.01\n",
      "204 50 -0.570260226727 -0.000112202587902 0.01\n",
      "204 100 -0.639184474945 -0.000112202587902 0.01\n",
      "204 150 -0.43128412962 -0.000112202587902 0.01\n",
      "204 200 -0.641816198826 -0.000112202587902 0.01\n",
      "204 250 -0.864787578583 -0.000112202587902 0.01\n",
      "204 300 -0.416613042355 -0.000112202587902 0.01\n",
      "204 350 -0.973035275936 -0.000112202587902 0.01\n",
      "204 400 -0.842298865318 -0.000112202587902 0.01\n",
      "204 450 -0.688042759895 -0.000112202587902 0.01\n",
      "204 500 -0.925442755222 -0.000112202587902 0.01\n",
      "204 550 -0.778053224087 -0.000112202587902 0.01\n",
      "valid_acc 80.25\n",
      "205 0 -0.503304421902 -0.000112202587902 0.01\n",
      "205 50 -0.640439093113 -0.000112202587902 0.01\n",
      "205 100 -1.55851125717 -0.000112202587902 0.01\n",
      "205 150 -1.33399522305 -0.000112202587902 0.01\n",
      "205 200 -0.529896438122 -0.000112202587902 0.01\n",
      "205 250 -0.542368471622 -0.000112202587902 0.01\n",
      "205 300 -0.475169837475 -0.000112202587902 0.01\n",
      "205 350 -1.14486181736 -0.000112202587902 0.01\n",
      "205 400 -0.644923627377 -0.000112202587902 0.01\n",
      "205 450 -0.908475100994 -0.000112202587902 0.01\n",
      "205 500 -0.747143149376 -0.000112202587902 0.01\n",
      "205 550 -0.874045789242 -0.000112202587902 0.01\n",
      "valid_acc 75.76666666666667\n",
      "206 0 -1.49157166481 -0.000112202587902 0.01\n",
      "206 50 -0.627909064293 -0.000112202587902 0.01\n",
      "206 100 -0.501034557819 -0.000112202587902 0.01\n",
      "206 150 -0.949877023697 -0.000112202587902 0.01\n",
      "206 200 -1.04334294796 -0.000112202587902 0.01\n",
      "206 250 -0.588027715683 -0.000112202587902 0.01\n",
      "206 300 -0.864603877068 -0.000112202587902 0.01\n",
      "206 350 -0.533645153046 -0.000112202587902 0.01\n",
      "206 400 -0.554195046425 -0.000112202587902 0.01\n",
      "206 450 -0.624621331692 -0.000112202587902 0.01\n",
      "206 500 -0.728499889374 -0.000112202587902 0.01\n",
      "206 550 -0.702755689621 -0.000112202587902 0.01\n",
      "valid_acc 74.51666666666667\n",
      "207 0 -0.775599718094 -0.000112202587902 0.01\n",
      "207 50 -0.424760550261 -0.000112202587902 0.01\n",
      "207 100 -0.698641002178 -0.000112202587902 0.01\n",
      "207 150 -0.679968655109 -0.000112202587902 0.01\n",
      "207 200 -0.756226062775 -0.000112202587902 0.01\n",
      "207 250 -1.1646207571 -0.000112202587902 0.01\n",
      "207 300 -0.813568651676 -0.000112202587902 0.01\n",
      "207 350 -0.564307391644 -0.000112202587902 0.01\n",
      "207 400 -0.731357753277 -0.000112202587902 0.01\n",
      "207 450 -0.525116562843 -0.000112202587902 0.01\n",
      "207 500 -0.634474277496 -0.000112202587902 0.01\n",
      "207 550 -0.335053563118 -0.000112202587902 0.01\n",
      "valid_acc 78.35499999999999\n",
      "208 0 -0.760218679905 -0.000112202587902 0.01\n",
      "208 50 -0.747699558735 -0.000112202587902 0.01\n",
      "208 100 -0.504083991051 -0.000112202587902 0.01\n",
      "208 150 -0.679676055908 -0.000112202587902 0.01\n",
      "208 200 -0.968112766743 -0.000112202587902 0.01\n",
      "208 250 -0.751894056797 -0.000112202587902 0.01\n",
      "208 300 -0.747391581535 -0.000112202587902 0.01\n",
      "208 350 -0.423175722361 -0.000112202587902 0.01\n",
      "208 400 -0.669076740742 -0.000112202587902 0.01\n",
      "208 450 -0.986276984215 -0.000112202587902 0.01\n",
      "208 500 -0.586956083775 -0.000112202587902 0.01\n",
      "208 550 -0.750431895256 -0.000112202587902 0.01\n",
      "valid_acc 79.42\n",
      "209 0 -0.617787122726 -0.000112202587902 0.01\n",
      "209 50 -0.633762657642 -0.000112202587902 0.01\n",
      "209 100 -0.630494713783 -0.000112202587902 0.01\n",
      "209 150 -1.00225114822 -0.000112202587902 0.01\n",
      "209 200 -0.730771720409 -0.000112202587902 0.01\n",
      "209 250 -0.70307546854 -0.000112202587902 0.01\n",
      "209 300 -0.839108526707 -0.000112202587902 0.01\n",
      "209 350 -0.748163282871 -0.000112202587902 0.01\n",
      "209 400 -0.643905162811 -0.000112202587902 0.01\n",
      "209 450 -0.724730551243 -0.000112202587902 0.01\n",
      "209 500 -1.37063205242 -0.000112202587902 0.01\n",
      "209 550 -0.897781670094 -0.000112202587902 0.01\n",
      "valid_acc 72.315\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "210 0 -0.685525655746 -0.000112202587902 0.01\n",
      "210 50 -0.579519927502 -0.000112202587902 0.01\n",
      "210 100 -0.603381276131 -0.000112202587902 0.01\n",
      "210 150 -0.487536042929 -0.000112202587902 0.01\n",
      "210 200 -0.699151694775 -0.000112202587902 0.01\n",
      "210 250 -1.11324477196 -0.000112202587902 0.01\n",
      "210 300 -0.303980916739 -0.000112202587902 0.01\n",
      "210 350 -0.390177637339 -0.000112202587902 0.01\n",
      "210 400 -1.00099658966 -0.000112202587902 0.01\n",
      "210 450 -0.921086072922 -0.000112202587902 0.01\n",
      "210 500 -0.559709727764 -0.000112202587902 0.01\n",
      "210 550 -0.539414763451 -0.000112202587902 0.01\n",
      "valid_acc 75.09333333333333\n",
      "211 0 -0.78613692522 -0.000112202587902 0.01\n",
      "211 50 -0.540627658367 -0.000112202587902 0.01\n",
      "211 100 -0.97925812006 -0.000112202587902 0.01\n",
      "211 150 -0.701588869095 -0.000112202587902 0.01\n",
      "211 200 -0.420407801867 -0.000112202587902 0.01\n",
      "211 250 -0.569962859154 -0.000112202587902 0.01\n",
      "211 300 -0.759323060513 -0.000112202587902 0.01\n",
      "211 350 -0.545364499092 -0.000112202587902 0.01\n",
      "211 400 -0.754109025002 -0.000112202587902 0.01\n",
      "211 450 -0.942687630653 -0.000112202587902 0.01\n",
      "211 500 -0.8612383008 -0.000112202587902 0.01\n",
      "211 550 -0.50555151701 -0.000112202587902 0.01\n",
      "valid_acc 78.60666666666667\n",
      "212 0 -0.599919378757 -0.000112202587902 0.01\n",
      "212 50 -0.608376443386 -0.000112202587902 0.01\n",
      "212 100 -0.628484666348 -0.000112202587902 0.01\n",
      "212 150 -0.924282968044 -0.000112202587902 0.01\n",
      "212 200 -0.636978149414 -0.000112202587902 0.01\n",
      "212 250 -0.478600233793 -0.000112202587902 0.01\n",
      "212 300 -0.655888974667 -0.000112202587902 0.01\n",
      "212 350 -0.602994143963 -0.000112202587902 0.01\n",
      "212 400 -1.36711192131 -0.000112202587902 0.01\n",
      "212 450 -0.675085902214 -0.000112202587902 0.01\n",
      "212 500 -0.533668994904 -0.000112202587902 0.01\n",
      "212 550 -0.939090371132 -0.000112202587902 0.01\n",
      "valid_acc 77.70833333333333\n",
      "213 0 -0.627843379974 -0.000112202587902 0.01\n",
      "213 50 -0.592064023018 -0.000112202587902 0.01\n",
      "213 100 -0.915436983109 -0.000112202587902 0.01\n",
      "213 150 -0.640289604664 -0.000112202587902 0.01\n",
      "213 200 -0.586170136929 -0.000112202587902 0.01\n",
      "213 250 -0.763897299767 -0.000112202587902 0.01\n",
      "213 300 -0.72586029768 -0.000112202587902 0.01\n",
      "213 350 -1.02473020554 -0.000112202587902 0.01\n",
      "213 400 -0.931157529354 -0.000112202587902 0.01\n",
      "213 450 -0.591722309589 -0.000112202587902 0.01\n",
      "213 500 -0.693534493446 -0.000112202587902 0.01\n",
      "213 550 -0.802077889442 -0.000112202587902 0.01\n",
      "valid_acc 78.28166666666667\n",
      "214 0 -0.911591768265 -0.000112202587902 0.01\n",
      "214 50 -0.821143805981 -0.000112202587902 0.01\n",
      "214 100 -0.985298991203 -0.000112202587902 0.01\n",
      "214 150 -0.694587647915 -0.000112202587902 0.01\n",
      "214 200 -0.885599195957 -0.000112202587902 0.01\n",
      "214 250 -0.527669012547 -0.000112202587902 0.01\n",
      "214 300 -0.71098023653 -0.000112202587902 0.01\n",
      "214 350 -0.676724076271 -0.000112202587902 0.01\n",
      "214 400 -0.759975492954 -0.000112202587902 0.01\n",
      "214 450 -0.467908859253 -0.000112202587902 0.01\n",
      "214 500 -0.888475894928 -0.000112202587902 0.01\n",
      "214 550 -0.855726242065 -0.000112202587902 0.01\n",
      "valid_acc 75.80833333333334\n",
      "215 0 -0.822496652603 -0.000112202587902 0.01\n",
      "215 50 -0.627565324306 -0.000112202587902 0.01\n",
      "215 100 -0.404854476452 -0.000112202587902 0.01\n",
      "215 150 -1.29378843307 -0.000112202587902 0.01\n",
      "215 200 -0.811153888702 -0.000112202587902 0.01\n",
      "215 250 -0.697184324265 -0.000112202587902 0.01\n",
      "215 300 -0.433609992266 -0.000112202587902 0.01\n",
      "215 350 -0.515462696552 -0.000112202587902 0.01\n",
      "215 400 -0.688618719578 -0.000112202587902 0.01\n",
      "215 450 -0.637578010559 -0.000112202587902 0.01\n",
      "215 500 -0.565018415451 -0.000112202587902 0.01\n",
      "215 550 -0.858866512775 -0.000112202587902 0.01\n",
      "valid_acc 73.70833333333333\n",
      "216 0 -0.616751492023 -0.000112202587902 0.01\n",
      "216 50 -0.585180103779 -0.000112202587902 0.01\n",
      "216 100 -0.466608136892 -0.000112202587902 0.01\n",
      "216 150 -0.724814355373 -0.000112202587902 0.01\n",
      "216 200 -0.931515574455 -0.000112202587902 0.01\n",
      "216 250 -0.721545696259 -0.000112202587902 0.01\n",
      "216 300 -0.675467967987 -0.000112202587902 0.01\n",
      "216 350 -0.612320244312 -0.000112202587902 0.01\n",
      "216 400 -0.896851956844 -0.000112202587902 0.01\n",
      "216 450 -0.687932372093 -0.000112202587902 0.01\n",
      "216 500 -0.661567389965 -0.000112202587902 0.01\n",
      "216 550 -0.620131254196 -0.000112202587902 0.01\n",
      "valid_acc 77.515\n",
      "217 0 -0.722369074821 -0.000112202587902 0.01\n",
      "217 50 -0.91146248579 -0.000112202587902 0.01\n",
      "217 100 -0.641819894314 -0.000112202587902 0.01\n",
      "217 150 -0.729925215244 -0.000112202587902 0.01\n",
      "217 200 -0.588878571987 -0.000112202587902 0.01\n",
      "217 250 -0.765411555767 -0.000112202587902 0.01\n",
      "217 300 -0.832302927971 -0.000112202587902 0.01\n",
      "217 350 -0.733961105347 -0.000112202587902 0.01\n",
      "217 400 -0.96570456028 -0.000112202587902 0.01\n",
      "217 450 -0.510095357895 -0.000112202587902 0.01\n",
      "217 500 -0.844844818115 -0.000112202587902 0.01\n",
      "217 550 -0.863951921463 -0.000112202587902 0.01\n",
      "valid_acc 77.24333333333333\n",
      "218 0 -0.651041209698 -0.000112202587902 0.01\n",
      "218 50 -0.66906785965 -0.000112202587902 0.01\n",
      "218 100 -0.735869169235 -0.000112202587902 0.01\n",
      "218 150 -0.484219133854 -0.000112202587902 0.01\n",
      "218 200 -0.555452048779 -0.000112202587902 0.01\n",
      "218 250 -0.656259715557 -0.000112202587902 0.01\n",
      "218 300 -0.551441669464 -0.000112202587902 0.01\n",
      "218 350 -0.71744209528 -0.000112202587902 0.01\n",
      "218 400 -1.06323254108 -0.000112202587902 0.01\n",
      "218 450 -0.871704876423 -0.000112202587902 0.01\n",
      "218 500 -0.499736636877 -0.000112202587902 0.01\n",
      "218 550 -0.900801539421 -0.000112202587902 0.01\n",
      "valid_acc 73.79\n",
      "219 0 -0.598254263401 -0.000112202587902 0.01\n",
      "219 50 -0.759383320808 -0.000112202587902 0.01\n",
      "219 100 -0.74325209856 -0.000112202587902 0.01\n",
      "219 150 -1.47052657604 -0.000112202587902 0.01\n",
      "219 200 -0.928113758564 -0.000112202587902 0.01\n",
      "219 250 -0.949907541275 -0.000112202587902 0.01\n",
      "219 300 -0.707539498806 -0.000112202587902 0.01\n",
      "219 350 -0.452430546284 -0.000112202587902 0.01\n",
      "219 400 -0.741917967796 -0.000112202587902 0.01\n",
      "219 450 -1.15848195553 -0.000112202587902 0.01\n",
      "219 500 -0.930138647556 -0.000112202587902 0.01\n",
      "219 550 -0.894169867039 -0.000112202587902 0.01\n",
      "valid_acc 79.98666666666666\n",
      "220 0 -0.498763442039 -0.000112202587902 0.01\n",
      "220 50 -0.86113679409 -0.000112202587902 0.01\n",
      "220 100 -0.543458759785 -0.000112202587902 0.01\n",
      "220 150 -0.456257522106 -0.000112202587902 0.01\n",
      "220 200 -0.77314889431 -0.000112202587902 0.01\n",
      "220 250 -0.436814039946 -0.000112202587902 0.01\n",
      "220 300 -0.754219770432 -0.000112202587902 0.01\n",
      "220 350 -0.987998962402 -0.000112202587902 0.01\n",
      "220 400 -0.766509532928 -0.000112202587902 0.01\n",
      "220 450 -0.810235917568 -0.000112202587902 0.01\n",
      "220 500 -0.853122532368 -0.000112202587902 0.01\n",
      "220 550 -0.552626013756 -0.000112202587902 0.01\n",
      "valid_acc 74.61333333333333\n",
      "221 0 -0.439780652523 -0.000112202587902 0.01\n",
      "221 50 -0.615744233131 -0.000112202587902 0.01\n",
      "221 100 -0.711040139198 -0.000112202587902 0.01\n",
      "221 150 -0.392257392406 -0.000112202587902 0.01\n",
      "221 200 -0.620743513107 -0.000112202587902 0.01\n",
      "221 250 -0.609644591808 -0.000112202587902 0.01\n",
      "221 300 -0.49466624856 -0.000112202587902 0.01\n",
      "221 350 -0.750509083271 -0.000112202587902 0.01\n",
      "221 400 -0.931622743607 -0.000112202587902 0.01\n",
      "221 450 -0.63251298666 -0.000112202587902 0.01\n",
      "221 500 -0.738489866257 -0.000112202587902 0.01\n",
      "221 550 -0.929129123688 -0.000112202587902 0.01\n",
      "valid_acc 79.81833333333334\n",
      "222 0 -0.685501277447 -0.000112202587902 0.01\n",
      "222 50 -0.792753994465 -0.000112202587902 0.01\n",
      "222 100 -0.775800943375 -0.000112202587902 0.01\n",
      "222 150 -0.668875038624 -0.000112202587902 0.01\n",
      "222 200 -1.12237501144 -0.000112202587902 0.01\n",
      "222 250 -0.772242963314 -0.000112202587902 0.01\n",
      "222 300 -0.431681722403 -0.000112202587902 0.01\n",
      "222 350 -0.690097510815 -0.000112202587902 0.01\n",
      "222 400 -1.07051968575 -0.000112202587902 0.01\n",
      "222 450 -0.540797472 -0.000112202587902 0.01\n",
      "222 500 -0.804795384407 -0.000112202587902 0.01\n",
      "222 550 -0.654874503613 -0.000112202587902 0.01\n",
      "valid_acc 78.83333333333333\n",
      "223 0 -0.753912031651 -0.000112202587902 0.01\n",
      "223 50 -0.725311458111 -0.000112202587902 0.01\n",
      "223 100 -0.925464451313 -0.000112202587902 0.01\n",
      "223 150 -0.956351518631 -0.000112202587902 0.01\n",
      "223 200 -0.793379366398 -0.000112202587902 0.01\n",
      "223 250 -0.621682405472 -0.000112202587902 0.01\n",
      "223 300 -0.602176845074 -0.000112202587902 0.01\n",
      "223 350 -0.572297990322 -0.000112202587902 0.01\n",
      "223 400 -1.03077363968 -0.000112202587902 0.01\n",
      "223 450 -0.696063518524 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "223 500 -0.87151992321 -0.000112202587902 0.01\n",
      "223 550 -0.718733847141 -0.000112202587902 0.01\n",
      "valid_acc 77.56666666666666\n",
      "224 0 -0.619056940079 -0.000112202587902 0.01\n",
      "224 50 -0.644972324371 -0.000112202587902 0.01\n",
      "224 100 -0.754298865795 -0.000112202587902 0.01\n",
      "224 150 -0.789782583714 -0.000112202587902 0.01\n",
      "224 200 -0.410754054785 -0.000112202587902 0.01\n",
      "224 250 -0.911396563053 -0.000112202587902 0.01\n",
      "224 300 -1.26628601551 -0.000112202587902 0.01\n",
      "224 350 -0.590975046158 -0.000112202587902 0.01\n",
      "224 400 -0.636246263981 -0.000112202587902 0.01\n",
      "224 450 -0.824734210968 -0.000112202587902 0.01\n",
      "224 500 -0.592872679234 -0.000112202587902 0.01\n",
      "224 550 -1.06981897354 -0.000112202587902 0.01\n",
      "valid_acc 74.58500000000001\n",
      "225 0 -0.853442013264 -0.000112202587902 0.01\n",
      "225 50 -0.894848585129 -0.000112202587902 0.01\n",
      "225 100 -0.962149679661 -0.000112202587902 0.01\n",
      "225 150 -0.439244002104 -0.000112202587902 0.01\n",
      "225 200 -0.730589091778 -0.000112202587902 0.01\n",
      "225 250 -0.596615970135 -0.000112202587902 0.01\n",
      "225 300 -0.788085579872 -0.000112202587902 0.01\n",
      "225 350 -0.70571243763 -0.000112202587902 0.01\n",
      "225 400 -0.847270727158 -0.000112202587902 0.01\n",
      "225 450 -0.665534079075 -0.000112202587902 0.01\n",
      "225 500 -0.616931319237 -0.000112202587902 0.01\n",
      "225 550 -0.610946297646 -0.000112202587902 0.01\n",
      "valid_acc 77.15166666666666\n",
      "226 0 -0.728042006493 -0.000112202587902 0.01\n",
      "226 50 -0.819324970245 -0.000112202587902 0.01\n",
      "226 100 -0.748478233814 -0.000112202587902 0.01\n",
      "226 150 -1.02672946453 -0.000112202587902 0.01\n",
      "226 200 -0.477953672409 -0.000112202587902 0.01\n",
      "226 250 -0.484806865454 -0.000112202587902 0.01\n",
      "226 300 -1.05403399467 -0.000112202587902 0.01\n",
      "226 350 -0.605602502823 -0.000112202587902 0.01\n",
      "226 400 -0.646946132183 -0.000112202587902 0.01\n",
      "226 450 -0.957110643387 -0.000112202587902 0.01\n",
      "226 500 -0.307176142931 -0.000112202587902 0.01\n",
      "226 550 -0.783967614174 -0.000112202587902 0.01\n",
      "valid_acc 77.75166666666667\n",
      "227 0 -0.438352167606 -0.000112202587902 0.01\n",
      "227 50 -0.456205606461 -0.000112202587902 0.01\n",
      "227 100 -0.635681867599 -0.000112202587902 0.01\n",
      "227 150 -0.516033947468 -0.000112202587902 0.01\n",
      "227 200 -1.41118514538 -0.000112202587902 0.01\n",
      "227 250 -0.813166558743 -0.000112202587902 0.01\n",
      "227 300 -0.590997755527 -0.000112202587902 0.01\n",
      "227 350 -0.610066652298 -0.000112202587902 0.01\n",
      "227 400 -0.69545096159 -0.000112202587902 0.01\n",
      "227 450 -0.706079304218 -0.000112202587902 0.01\n",
      "227 500 -0.722207546234 -0.000112202587902 0.01\n",
      "227 550 -0.746636807919 -0.000112202587902 0.01\n",
      "valid_acc 78.17166666666667\n",
      "228 0 -0.375495642424 -0.000112202587902 0.01\n",
      "228 50 -0.834200501442 -0.000112202587902 0.01\n",
      "228 100 -0.668046593666 -0.000112202587902 0.01\n",
      "228 150 -0.63825917244 -0.000112202587902 0.01\n",
      "228 200 -0.774082779884 -0.000112202587902 0.01\n",
      "228 250 -0.804499208927 -0.000112202587902 0.01\n",
      "228 300 -0.705879569054 -0.000112202587902 0.01\n",
      "228 350 -1.03772461414 -0.000112202587902 0.01\n",
      "228 400 -0.454039186239 -0.000112202587902 0.01\n",
      "228 450 -0.667872488499 -0.000112202587902 0.01\n",
      "228 500 -0.592854559422 -0.000112202587902 0.01\n",
      "228 550 -0.604674458504 -0.000112202587902 0.01\n",
      "valid_acc 78.25\n",
      "229 0 -0.635021150112 -0.000112202587902 0.01\n",
      "229 50 -0.6738910079 -0.000112202587902 0.01\n",
      "229 100 -0.99037116766 -0.000112202587902 0.01\n",
      "229 150 -0.810207188129 -0.000112202587902 0.01\n",
      "229 200 -0.826241910458 -0.000112202587902 0.01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-565:\n",
      "KeyboardInterrupt\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-50-787f2ca587f6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m       \u001b[0mupdate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msolutions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m       \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m       \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m       \u001b[0mreward\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/hadavid/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_pre_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m             \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    225\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-39-21c862646013>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     16\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_filter2\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m)\u001b[0m   \u001b[0;31m# reshape Variable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/hadavid/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mrelu\u001b[0;34m(input, inplace)\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 463\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mThreshold\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    464\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 34, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/home/hadavid/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n"
     ]
    }
   ],
   "source": [
    "#'''\n",
    "best_valid_acc = 0\n",
    "training_log = []\n",
    "for epoch in range(1, 10*args.epochs + 1):\n",
    "\n",
    "  # train loop\n",
    "  model.eval()\n",
    "  for batch_idx, (data, target) in enumerate(train_loader):\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data), Variable(target)\n",
    "    \n",
    "    solutions = es.ask()\n",
    "    reward = np.zeros(es.popsize)\n",
    "    \n",
    "    for i in range(es.popsize):\n",
    "      update_model(solutions[i], model, model_shapes)\n",
    "      output = model(data)\n",
    "      loss = F.nll_loss(output, target)\n",
    "      reward[i] = - loss.data[0]\n",
    "\n",
    "    best_raw_reward = reward.max()\n",
    "    reward = compute_centered_ranks(reward)\n",
    "    l2_decay = compute_weight_decay(weight_decay_coef, solutions)\n",
    "    reward += l2_decay\n",
    "\n",
    "    es.tell(reward)\n",
    "\n",
    "    result = es.result()\n",
    "    \n",
    "    if (batch_idx % 50 == 0):\n",
    "      print(epoch, batch_idx, best_raw_reward, result[0].mean(), result[3])\n",
    "\n",
    "  curr_solution = es.current_param()\n",
    "  update_model(curr_solution, model, model_shapes)\n",
    "\n",
    "  valid_acc = evaluate(model, valid_loader, print_mode=False)\n",
    "  training_log.append([epoch, valid_acc])\n",
    "  print('valid_acc', valid_acc * 100.)\n",
    "  if valid_acc >= best_valid_acc:\n",
    "    best_valid_acc = valid_acc\n",
    "    best_model = copy.deepcopy(model)\n",
    "    print('best valid_acc', best_valid_acc * 100.)\n",
    "#'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(best_model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(best_model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(best_model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.best_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.current_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eval_acc = evaluate(best_model, test_loader)\n",
    "print('final test acc', eval_acc * 100.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_count = 0\n",
    "for param in model.parameters():\n",
    "  print(param.data.shape)\n",
    "  param_count += np.product(param.data.shape)\n",
    "print(param_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params = []\n",
    "for param in orig_model.parameters():\n",
    "  orig_params.append(param.data.cpu().numpy().flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params_flat = np.concatenate(orig_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_ = plt.hist(orig_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_params = []\n",
    "for param in best_model.parameters():\n",
    "  final_params.append(param.data.cpu().numpy().flatten())\n",
    "final_params_flat = np.concatenate(final_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_ = plt.hist(final_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
